Marmelator/aho-corasick
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

use word transitions for whole word only mode

Browse Source
This commit is contained in:
Douglas Lovell 2015-11-02 17:14:28 -07:00
parent f89b000894
commit d764017abe
13 changed files with 120 additions and 156 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/main/java/org/ahocorasick/trie/CharacterTransition.java
View File

@ -29,9 +29,4 @@ class CharacterTransition extends Transition<Character> {
this(c, 0);
}
@Override
public boolean isWordSeparator() {
return Character.isWhitespace(token);
}
}

13
src/main/java/org/ahocorasick/trie/Emit.java
View File

@ -6,27 +6,16 @@ import org.ahocorasick.interval.Intervalable;
public class Emit extends Interval implements Intervalable {
private final String keyword;
private final boolean isWholeWord;
public Emit(final int start, final int end,
final String keyword, boolean isWholeWord) {
public Emit(final int start, final int end, final String keyword) {
super(start, end);
this.keyword = keyword;
this.isWholeWord = isWholeWord;
}
public Emit(final int start, final int end, final String keyword) {
this(start, end, keyword, true);
}
public String getKeyword() {
return this.keyword;
}
public boolean isWholeWord() {
return isWholeWord;
}
@Override
public String toString() {
return super.toString() + "=" + this.keyword;

13
src/main/java/org/ahocorasick/trie/FragmentToken.java
View File

@ -2,16 +2,8 @@ package org.ahocorasick.trie;
public class FragmentToken extends Token {
private boolean whiteSpace;
public FragmentToken(String fragment) {
super(fragment);
this.whiteSpace = true;
for (int position = 0; position < fragment.length(); position++) {
if (!Character.isWhitespace(fragment.charAt(position))) {
whiteSpace = false;
}
}
}
@Override
@ -24,9 +16,4 @@ public class FragmentToken extends Token {
return null;
}
@Override
public boolean isWhiteSpace() {
return whiteSpace;
}
}

54
src/main/java/org/ahocorasick/trie/Keyword.java Normal file
View File

@ -0,0 +1,54 @@
/*
* Copyright 2015 Rogue Wave Software.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.ahocorasick.trie;
/**
*
* @author doug.lovell
*/
public class Keyword implements Comparable {
private final String text;
private int depth;
public Keyword(String text, int depth) {
this.text = text;
this.depth = depth;
}
public void setDepth(int depth) {
this.depth = depth;
}
public int getDepth() {
return depth;
}
public String getText() {
return text;
}
public String toString() {
return "Keyword '" + text + "' at depth " + depth;
}
@Override
public int compareTo(Object o) {
if (o instanceof Keyword) {
return text.compareTo(((Keyword) o).text);
}
throw new IllegalArgumentException("Only supports comparison with other keywords");
}
}

5
src/main/java/org/ahocorasick/trie/MatchToken.java
View File

@ -9,11 +9,6 @@ public class MatchToken extends Token {
this.emit = emit;
}
@Override
public boolean isWholeWord() {
return emit.isWholeWord();
}
@Override
public boolean isMatch() {
return true;

16
src/main/java/org/ahocorasick/trie/State.java
View File

@ -43,7 +43,7 @@ public class State {
private State failure = null;
/** whenever this state is reached, it will emit the matches keywords for future reference */
private Set<String> emits = null;
private Set<Keyword> emits = null;
public State() {
this(0);
@ -83,21 +83,25 @@ public class State {
return this.depth;
}
public void addEmit(String keyword) {
public void addEmit(Keyword keyword) {
if (this.emits == null) {
this.emits = new TreeSet<>();
}
this.emits.add(keyword);
}
public void addEmit(Collection<String> emits) {
for (String emit : emits) {
public void addEmit(Collection<Keyword> emits) {
for (Keyword emit : emits) {
addEmit(emit);
}
}
public void addEmitString(String key) {
addEmit(new Keyword(key, depth));
}
public Collection<String> emit() {
return this.emits == null ? Collections.<String> emptyList() : this.emits;
public Collection<Keyword> emit() {
return this.emits == null ? Collections.<Keyword> emptyList() : this.emits;
}
public State failure(EmitCandidateFlushHandler emitCandidateFlushHandler) {

10
src/main/java/org/ahocorasick/trie/Token.java
View File

@ -2,7 +2,7 @@ package org.ahocorasick.trie;
public abstract class Token {
private String fragment;
private final String fragment;
public Token(String fragment) {
this.fragment = fragment;
@ -14,14 +14,6 @@ public abstract class Token {
public abstract boolean isMatch();
public boolean isWholeWord() {
return false;
}
public boolean isWhiteSpace() {
return false;
}
public abstract Emit getEmit();
}

2
src/main/java/org/ahocorasick/trie/Tokenizer.java
View File

@ -9,7 +9,7 @@ public class Tokenizer {
private final Collection<Emit> emits;
private final String text;
public Tokenizer(Collection<Emit> emits, String text) {
this.emits = emits;
this.text = text;

7
src/main/java/org/ahocorasick/trie/Transition.java
View File

@ -23,7 +23,7 @@ import java.util.Objects;
* @author doug.lovell
* @param <T>
*/
public abstract class Transition<T> {
public class Transition<T> {
protected final T token;
protected final int start;
protected final int length;
@ -46,11 +46,10 @@ public abstract class Transition<T> {
return length;
}
public abstract boolean isWordSeparator();
@Override
public String toString() {
return "Transition on " + token;
return "Transition on '" + token + "' start: " + start +
", length: " + length;
}
@Override

97
src/main/java/org/ahocorasick/trie/Trie.java
View File

@ -10,6 +10,8 @@ import org.ahocorasick.trie.handler.FirstMatchHandler;
import java.util.Collection;
import java.util.Queue;
import java.util.LinkedList;
import java.util.ListIterator;
import java.util.concurrent.LinkedBlockingDeque;
/**
@ -43,34 +45,16 @@ public class Trie {
public int getPosition() {
return position;
}
public abstract Emit match(Keyword kwd, int start, int position);
}
private class WordTokenizer extends KeywordTokenizer {
public WordTokenizer(CharSequence input) {
super(input);
System.out.println("WORDTOKENIZER input '" + input + "'");
// leading and trailing white space cannot be part of the
// search pattern
int start = 0;
while (start < length && Character.isWhitespace(input.charAt(start))) {
++start;
}
int end = length - 1;
while (start < end && Character.isWhitespace(input.charAt(end))) {
--end;
}
this.input = input.subSequence(start, end + 1);
this.length = end - start + 1;
System.out.println("WORDTOKENIZER input '" + this.input + "'");
System.out.println("input.length " + this.input.length() + ", this.length " + this.length);
}
@Override
public Transition<String> nextTransition() {
WordTransition t = null;
System.out.println("WORDTOKENIZER get next word transition");
System.out.println("Position: " + position);
System.out.println("Text under cursor, '" +
input.subSequence(Math.min(length-1, position), Math.min(length, position + 10)) + "'");
while (position < length && Character.isWhitespace(currentChar())) {
++position;
}
@ -82,10 +66,17 @@ public class Trie {
String word = input.subSequence(start, position).toString();
t = new WordTransition(word, start);
}
System.out.println("New position: " + position);
System.out.println("Text in transition, '" + (t == null ? "null" : t.transitionToken()) + "'");
return t;
}
/*
On word matching, we return the matched text, which can be of different
length that the keyword, due to whitespace differences.
*/
@Override
public Emit match(Keyword kwd, int start, int position) {
String matchedText = input.subSequence(start, position).toString();
return new Emit(start, position - 1, matchedText);
}
}
private class CharacterTokenizer extends KeywordTokenizer {
@ -95,17 +86,20 @@ public class Trie {
@Override
public Transition<Character> nextTransition() {
CharacterTransition t = null;
System.out.println("CHARACTERTOKENIZER get next character transition");
System.out.println("Position: " + position);
System.out.println("Text under cursor, '" + input.subSequence(position, Math.min(length, position + 10)) + "'");
if (position < length) {
t = new CharacterTransition(currentChar(), position);
position += 1;
}
System.out.println("New position: " + position);
System.out.println("Text in transition, '" + (t == null ? "null" : t.transitionToken()) + "'");
return t;
}
/*
On character matching, the tests expect the implementation to
return the matched keyword.
*/
@Override
public Emit match(Keyword kwd, int start, int position) {
return new Emit(start, position - 1, kwd.getText());
}
}
private class TokenStream {
@ -120,7 +114,7 @@ public class Trie {
input.append(trieConfig.isCaseInsensitive() ?
Character.toLowerCase(ch) : ch);
}
if (trieConfig.hasWordTransitions()) {
if (trieConfig.isOnlyWholeWords()) {
kwt = new WordTokenizer(input);
}
else {
@ -140,18 +134,14 @@ public class Trie {
return next;
}
public boolean isWholeWord(int start) {
if (lookahead == null) {
lookahead = kwt.nextTransition();
}
return ((start == 0 ||
Character.isWhitespace(input.charAt(start-1))) &&
(lookahead == null || lookahead.isWordSeparator()));
}
public String input() {
return input.toString();
}
public Emit match(Keyword kwd, int start, int position) {
return kwt.match(kwd, start, position);
}
}
private void addKeyword(CharSequence keyword) {
@ -165,7 +155,7 @@ public class Trie {
currentState = currentState.addState(tn);
tn = tknz.nextTransition();
}
currentState.addEmit(tknz.input());
currentState.addEmitString(tknz.input());
}
public Collection<Token> tokenize(String text) {
@ -201,27 +191,25 @@ public class Trie {
TokenStream tknz = new TokenStream(text);
LinkedList<Transition> tknHistory = new LinkedList<>();
State currentState = this.rootState;
Transition tn = tknz.nextTransition();
while (tn != null) {
if (flushHandler.stop()) {
return;
}
tknHistory.add(tn);
currentState = getState(currentState, tn, flushHandler);
Collection<String> emits = currentState.emit();
for (String emit : emits) {
Collection<Keyword> emits = currentState.emit();
int depth = currentState.getDepth();
while (depth < tknHistory.size()) {
tknHistory.remove();
}
for (Keyword emit : emits) {
int position = tn.getStart() + tn.getLength();
int start = position - emit.length();
if (start < 0) {
System.out.println("START < 0 !! at " + position + " on '" + emit + "', length " + emit.length());
System.out.println("TEXT is '" + text + "'");
System.out.println(tn);
}
boolean isWholeWord = tknz.isWholeWord(start);
if (isWholeWord || !trieConfig.isOnlyWholeWords()) {
emitCandidateHolder.addCandidate(
new Emit(start, position - 1, emit, isWholeWord));
}
int start = tknHistory.get(depth - emit.getDepth()).getStart();
ListIterator<Transition> tns = tknHistory.listIterator();
emitCandidateHolder.addCandidate(tknz.match(emit, start, position));
}
tn = tknz.nextTransition();
}
@ -290,16 +278,11 @@ public class Trie {
}
public TrieBuilder onlyWholeWords() {
this.trieConfig.setOnlyWholeWords(true);
return this;
}
public TrieBuilder wordTransitions() {
if (hasAddedKeyword) {
throw new IllegalStateException(
"Unable to switch to word transitions after keywords added");
"Unable to switch to only whole words after keywords added");
}
this.trieConfig.setWordTransitions(true);
this.trieConfig.setOnlyWholeWords(true);
return this;
}

10
src/main/java/org/ahocorasick/trie/TrieConfig.java
View File

@ -8,8 +8,6 @@ public class TrieConfig {
private boolean caseInsensitive = false;
private boolean wordTransitions = false;
public boolean isAllowOverlaps() {
return allowOverlaps;
}
@ -33,12 +31,4 @@ public class TrieConfig {
public void setCaseInsensitive(boolean caseInsensitive) {
this.caseInsensitive = caseInsensitive;
}
public boolean hasWordTransitions() {
return wordTransitions;
}
public void setWordTransitions(boolean wordNodes) {
this.wordTransitions = wordNodes;
}
}

5
src/main/java/org/ahocorasick/trie/WordTransition.java
View File

@ -28,9 +28,4 @@ public class WordTransition extends Transition<String> {
public WordTransition(String s) {
this(s, 0);
}
@Override
public boolean isWordSeparator() {
return true;
}
}

39
src/test/java/org/ahocorasick/trie/TrieTest.java
View File

@ -81,7 +81,7 @@ public class TrieTest {
@Test
public void variousKeywordsFirstMatchWordTransitions() {
Trie trie = Trie.builder()
.wordTransitions()
.onlyWholeWords()
.addKeyword("abc")
.addKeyword("bcd")
.addKeyword("cde")
@ -198,7 +198,7 @@ public class TrieTest {
@Test
public void recipesWordTransitions() {
Trie trie = Trie.builder()
.wordTransitions()
.onlyWholeWords()
.addKeyword("veal")
.addKeyword("cauliflower")
.addKeyword("broccoli")
@ -265,7 +265,7 @@ public class TrieTest {
public void nonOverlappingWordTransitions() {
Trie trie = Trie.builder()
.removeOverlaps()
.wordTransitions()
.onlyWholeWords()
.addKeyword("peper molen")
.addKeyword("molen wiel")
.addKeyword("wiel dop")
@ -436,7 +436,7 @@ public class TrieTest {
@Test
public void tokenizeFullSentenceByWords() {
Trie trie = Trie.builder()
.wordTransitions()
.onlyWholeWords()
.addKeyword("Alpha")
.addKeyword("Beta")
.addKeyword("Gamma")
@ -457,13 +457,13 @@ public class TrieTest {
public ExpectedException thrown = ExpectedException.none();
@Test
public void wordTransitionsThrowsExceptionAfterKeywordsAdded()
public void onlyWholeWordsThrowsExceptionAfterKeywordsAdded()
throws IllegalStateException {
thrown.expect(IllegalStateException.class);
thrown.expectMessage("Unable to switch to word transitions after keywords added");
thrown.expectMessage("Unable to switch to only whole words after keywords added");
Trie trie = Trie.builder()
.addKeyword("Happy for now")
.wordTransitions()
.onlyWholeWords()
.addKeyword("Not so happy")
.build();
}
@ -589,46 +589,27 @@ public class TrieTest {
checkEmit(emits.iterator().next(), 0, 9, "#sugar-123");
}
/*
What does "onlyWholeWords" mean when the keyword itself has spaces?
@Test
public void spacesAroundKeyword() {
String keyword = " lorem ipso facto genera linden pharma six 1 ";
Trie trie = Trie.builder()
.onlyWholeWords()
.caseInsensitive()
.addKeyword(keyword)
.build();
Collection < Emit > emits = trie.parseText(
"gravita conundrum" + keyword + "under addressed object ");
assertEquals(1, emits.size());
checkEmit(emits.iterator().next(), 0, keyword.length() + 1, keyword);
}
*/
/*
For wordTransitions, we'll ignore leading and trailing white space
For onlyWholeWords, we'll ignore leading and trailing white space
included on keywords
*/
@Test
public void spacesAroundKeywordByWords() {
String keyword = "lorem ipso facto genera linden pharma six 1";
Trie trie = Trie.builder()
.wordTransitions()
.onlyWholeWords()
.caseInsensitive()
.addKeyword(" " + keyword + " ")
.build();
Collection < Emit > emits = trie.parseText(
keyword + " under addressed object ");
assertEquals(1, emits.size());
checkEmit(emits.iterator().next(), 0, keyword.length(), keyword);
checkEmit(emits.iterator().next(), 0, keyword.length() - 1, keyword);
}
private void assertToken(Token token, String fragment, boolean match, boolean wholeWord, boolean whiteSpace) {
assertEquals(fragment, token.getFragment());
assertEquals(match, token.isMatch());
assertEquals(wholeWord, token.isWholeWord());
assertEquals(whiteSpace, token.isWhiteSpace());
}
private void checkEmit(Emit next, int expectedStart, int expectedEnd, String expectedKeyword) {