diff --git a/src/main/java/org/ahocorasick/trie/CharacterTransition.java b/src/main/java/org/ahocorasick/trie/CharacterTransition.java index 31bf4e8..b49019a 100644 --- a/src/main/java/org/ahocorasick/trie/CharacterTransition.java +++ b/src/main/java/org/ahocorasick/trie/CharacterTransition.java @@ -21,13 +21,12 @@ package org.ahocorasick.trie; */ class CharacterTransition extends Transition { - public CharacterTransition(Character c) { - super(c); + public CharacterTransition(Character c, int start) { + super(c, start, 1); } - @Override - public void updateMatch(StringBuilder match) { - match.append(token); + public CharacterTransition(Character c) { + this(c, 0); } @Override diff --git a/src/main/java/org/ahocorasick/trie/Transition.java b/src/main/java/org/ahocorasick/trie/Transition.java index d20ecd6..84e15ca 100644 --- a/src/main/java/org/ahocorasick/trie/Transition.java +++ b/src/main/java/org/ahocorasick/trie/Transition.java @@ -25,16 +25,27 @@ import java.util.Objects; */ public abstract class Transition { protected final T token; + protected final int start; + protected final int length; - public Transition(T token) { + public Transition(T token, int start, int length) { this.token = token; + this.start = start; + this.length = length; } public T transitionToken() { return token; } - public abstract void updateMatch(StringBuilder match); + public int getStart() { + return start; + } + + public int getLength() { + return length; + } + public abstract boolean isWordSeparator(); @Override diff --git a/src/main/java/org/ahocorasick/trie/Trie.java b/src/main/java/org/ahocorasick/trie/Trie.java index b6d3ab0..d285382 100644 --- a/src/main/java/org/ahocorasick/trie/Trie.java +++ b/src/main/java/org/ahocorasick/trie/Trie.java @@ -31,11 +31,15 @@ public class Trie { this.rootState = new State(); } - private interface KeywordTokenizer { - public Transition nextTransition(); + private abstract class KeywordTokenizer { + protected int position = 0; + public abstract Transition nextTransition(); + public int getPosition() { + return position; + } } - private class WordTokenizer implements KeywordTokenizer { + private class WordTokenizer extends KeywordTokenizer { private final Iterator st; public WordTokenizer(String keyword) { String[] tokens = keyword.split("\\s"); @@ -45,13 +49,18 @@ public class Trie { public Transition nextTransition() { WordTransition t = null; if (st.hasNext()) { - t = new WordTransition(st.next()); + String word = st.next(); + t = new WordTransition(word, position); + if (0 < position) { + position += 1; // a space + } + position += word.length(); } return t; } } - private class CharacterTokenizer implements KeywordTokenizer { + private class CharacterTokenizer extends KeywordTokenizer { private final java.text.StringCharacterIterator ct; private char cur; public CharacterTokenizer(String keyword) { @@ -62,9 +71,10 @@ public class Trie { public Transition nextTransition() { CharacterTransition t = null; if (cur != CharacterIterator.DONE) { - t = new CharacterTransition(cur); + t = new CharacterTransition(cur, position); cur = ct.next(); } + position += 1; return t; } } @@ -82,11 +92,18 @@ public class Trie { private class TokenStream { private final KeywordTokenizer kwt; + private final String input; private Transition lookahead; - private final StringBuilder match = new StringBuilder(); - public TokenStream(KeywordTokenizer kwt) { - this.kwt = kwt; + public TokenStream(String input) { + this.input = input; + if (trieConfig.hasWordTransitions()) { + kwt = new WordTokenizer(input); + } + else { + kwt = new CharacterTokenizer(input); + } + lookahead = null; } public Transition nextTransition() { @@ -97,22 +114,15 @@ public class Trie { else { lookahead = null; } - if (next != null) { - next.updateMatch(match); - } return next; } - public int position() { - return match.length() - 1; - } - public boolean isWholeWord(int start) { if (lookahead == null) { lookahead = kwt.nextTransition(); } return ((start == 0 || - Character.isSpaceChar(match.codePointAt(start-1))) && + Character.isSpaceChar(input.codePointAt(start-1))) && (lookahead == null || lookahead.isWordSeparator())); } } @@ -169,7 +179,7 @@ public class Trie { if (trieConfig.isCaseInsensitive()) { input = input.toLowerCase(); } - TokenStream tknz = new TokenStream(keywordTokenizer(input)); + TokenStream tknz = new TokenStream(input); State currentState = this.rootState; Transition tn = tknz.nextTransition(); @@ -180,7 +190,7 @@ public class Trie { currentState = getState(currentState, tn, flushHandler); Collection emits = currentState.emit(); for (String emit : emits) { - int position = tknz.position(); + int position = tn.getStart() + tn.getLength() - 1; int start = position - emit.length() + 1; boolean isWholeWord = tknz.isWholeWord(start); if (isWholeWord || !trieConfig.isOnlyWholeWords()) { diff --git a/src/main/java/org/ahocorasick/trie/WordTransition.java b/src/main/java/org/ahocorasick/trie/WordTransition.java index 1d631b5..e4f6436 100644 --- a/src/main/java/org/ahocorasick/trie/WordTransition.java +++ b/src/main/java/org/ahocorasick/trie/WordTransition.java @@ -21,21 +21,16 @@ package org.ahocorasick.trie; */ public class WordTransition extends Transition { - public WordTransition(String s) { - super(s); + public WordTransition(String s, int start) { + super(s, start, s.length()); } - @Override - public void updateMatch(StringBuilder match) { - if (0 < match.length()) { - match.append(' '); - } - match.append(token); + public WordTransition(String s) { + this(s, 0); } @Override public boolean isWordSeparator() { return true; } - }