put text positions on the transitions and track in the token stream

src/main/java/org/ahocorasick/trie/CharacterTransition.java

@@ -21,13 +21,12 @@ package org.ahocorasick.trie;
  */
 class CharacterTransition extends Transition<Character> {
 
-    public CharacterTransition(Character c) {
-        super(c);
+    public CharacterTransition(Character c, int start) {
+        super(c, start, 1);
     }
 
-    @Override
-    public void updateMatch(StringBuilder match) {
-        match.append(token);
+    public CharacterTransition(Character c) {
+        this(c, 0);
     }
 
     @Override

src/main/java/org/ahocorasick/trie/Transition.java

@@ -25,16 +25,27 @@ import java.util.Objects;
  */
 public abstract class Transition<T> {
     protected final T token;
+    protected final int start;
+    protected final int length;
     
-    public Transition(T token) {
+    public Transition(T token, int start, int length) {
         this.token = token;
+        this.start = start;
+        this.length = length;
     }
     
     public T transitionToken() {
         return token;
     }
     
-    public abstract void updateMatch(StringBuilder match);
+    public int getStart() {
+        return start;
+    }
+    
+    public int getLength() {
+        return length;
+    }
+    
     public abstract boolean isWordSeparator();
     
     @Override

src/main/java/org/ahocorasick/trie/Trie.java

@@ -31,11 +31,15 @@ public class Trie {
         this.rootState = new State();
     }
     
-    private interface KeywordTokenizer {
-        public Transition nextTransition();
+    private abstract class KeywordTokenizer {
+        protected int position = 0;
+        public abstract Transition nextTransition();
+        public int getPosition() {
+            return position;
+        }
     }
     
-    private class WordTokenizer implements KeywordTokenizer {
+    private class WordTokenizer extends KeywordTokenizer {
         private final Iterator<String> st;
         public WordTokenizer(String keyword) {
             String[] tokens = keyword.split("\\s");
@@ -45,13 +49,18 @@ public class Trie {
         public Transition<String> nextTransition() {
             WordTransition t = null;
             if (st.hasNext()) {
-                t = new WordTransition(st.next());
+                String word = st.next();
+                t = new WordTransition(word, position);
+                if (0 < position) {
+                    position += 1; // a space
+                }
+                position += word.length();
             }
             return t;
         }
     }
     
-    private class CharacterTokenizer implements KeywordTokenizer {
+    private class CharacterTokenizer extends KeywordTokenizer {
         private final java.text.StringCharacterIterator ct;
         private char cur;
         public CharacterTokenizer(String keyword) {
@@ -62,9 +71,10 @@ public class Trie {
         public Transition<Character> nextTransition() {
             CharacterTransition t = null;
             if (cur != CharacterIterator.DONE) {
-                t = new CharacterTransition(cur);
+                t = new CharacterTransition(cur, position);
                 cur = ct.next();
             }
+            position += 1;
             return t;
         }
     }
@@ -82,11 +92,18 @@ public class Trie {
 
     private class TokenStream {
         private final KeywordTokenizer kwt;
+        private final String input;
         private Transition lookahead;
-        private final StringBuilder match = new StringBuilder();
         
-        public TokenStream(KeywordTokenizer kwt) {
-            this.kwt = kwt;
+        public TokenStream(String input) {
+            this.input = input;
+            if (trieConfig.hasWordTransitions()) {
+                kwt = new WordTokenizer(input);
+            }
+            else {
+                kwt = new CharacterTokenizer(input);
+            }
+            lookahead = null;
         }
         
         public Transition nextTransition() {
@@ -97,22 +114,15 @@ public class Trie {
             else {
                 lookahead = null;
             }
-            if (next != null) {
-                next.updateMatch(match);
-            }
             return next;
         }
         
-        public int position() {
-            return match.length() - 1;
-        }
-        
         public boolean isWholeWord(int start) {
             if (lookahead == null) {
                 lookahead = kwt.nextTransition();
             }
             return ((start == 0 || 
-                     Character.isSpaceChar(match.codePointAt(start-1))) && 
+                     Character.isSpaceChar(input.codePointAt(start-1))) && 
                     (lookahead == null || lookahead.isWordSeparator()));
         }
     }
@@ -169,7 +179,7 @@ public class Trie {
         if (trieConfig.isCaseInsensitive()) {
             input = input.toLowerCase();
         }
-        TokenStream tknz = new TokenStream(keywordTokenizer(input));
+        TokenStream tknz = new TokenStream(input);
         
         State currentState = this.rootState;
         Transition tn = tknz.nextTransition();
@@ -180,7 +190,7 @@ public class Trie {
             currentState = getState(currentState, tn, flushHandler);
             Collection<String> emits = currentState.emit();
             for (String emit : emits) {
-                int position = tknz.position();
+                int position = tn.getStart() + tn.getLength() - 1;
                 int start = position - emit.length() + 1;
                 boolean isWholeWord = tknz.isWholeWord(start);
                 if (isWholeWord || !trieConfig.isOnlyWholeWords()) {

src/main/java/org/ahocorasick/trie/WordTransition.java

@@ -21,21 +21,16 @@ package org.ahocorasick.trie;
  */
 public class WordTransition extends Transition<String> {
 
-    public WordTransition(String s) {
-        super(s);
+    public WordTransition(String s, int start) {
+        super(s, start, s.length());
     }
 
-    @Override
-    public void updateMatch(StringBuilder match) {
-        if (0 < match.length()) {
-            match.append(' ');
-        }
-        match.append(token);
+    public WordTransition(String s) {
+        this(s, 0);
     }
 
     @Override
     public boolean isWordSeparator() {
         return true;
     }
-    
 }