use word transitions for whole word only mode

src/main/java/org/ahocorasick/trie/CharacterTransition.java

@@ -29,9 +29,4 @@ class CharacterTransition extends Transition<Character> {
         this(c, 0);
     }
 
-    @Override
-    public boolean isWordSeparator() {
-        return Character.isWhitespace(token);
-    }
-    
 }

src/main/java/org/ahocorasick/trie/Emit.java

@@ -6,27 +6,16 @@ import org.ahocorasick.interval.Intervalable;
 public class Emit extends Interval implements Intervalable {
 
     private final String keyword;
-    private final boolean isWholeWord;
-
-    public Emit(final int start, final int end, 
-            final String keyword, boolean isWholeWord) {
-        super(start, end);
-        this.keyword = keyword;
-        this.isWholeWord = isWholeWord;
-    }
 
     public Emit(final int start, final int end, final String keyword) {
-        this(start, end, keyword, true);
+        super(start, end);
+        this.keyword = keyword;
     }
     
     public String getKeyword() {
         return this.keyword;
     }
     
-    public boolean isWholeWord() {
-        return isWholeWord;
-    }
-
     @Override
     public String toString() {
         return super.toString() + "=" + this.keyword;

src/main/java/org/ahocorasick/trie/FragmentToken.java

@@ -2,16 +2,8 @@ package org.ahocorasick.trie;
 
 public class FragmentToken extends Token {
 
-    private boolean whiteSpace;
-
     public FragmentToken(String fragment) {
         super(fragment);
-        this.whiteSpace = true;
-        for (int position = 0; position < fragment.length(); position++) {
-            if (!Character.isWhitespace(fragment.charAt(position))) {
-                whiteSpace = false;
-            }
-        }
     }
 
     @Override
@@ -24,9 +16,4 @@ public class FragmentToken extends Token {
         return null;
     }
 
-    @Override
-    public boolean isWhiteSpace() {
-        return whiteSpace;
-    }
-
 }

src/main/java/org/ahocorasick/trie/Keyword.java

@@ -0,0 +1,54 @@
+/*
+ * Copyright 2015 Rogue Wave Software.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.ahocorasick.trie;
+
+/**
+ *
+ * @author doug.lovell
+ */
+public class Keyword implements Comparable {
+    private final String text;
+    private int depth;
+    
+    public Keyword(String text, int depth) {
+        this.text = text;
+        this.depth = depth;
+    }
+    
+    public void setDepth(int depth) {
+        this.depth = depth;
+    }
+    
+    public int getDepth() {
+        return depth;
+    }
+    
+    public String getText() {
+        return text;
+    }
+    
+    public String toString() {
+        return "Keyword '" + text + "' at depth " + depth;
+    }
+
+    @Override
+    public int compareTo(Object o) {
+        if (o instanceof Keyword) {
+            return text.compareTo(((Keyword) o).text);
+        }
+        throw new IllegalArgumentException("Only supports comparison with other keywords");
+    }
+}

src/main/java/org/ahocorasick/trie/MatchToken.java

@@ -9,11 +9,6 @@ public class MatchToken extends Token {
         this.emit = emit;
     }
 
-    @Override
-    public boolean isWholeWord() {
-        return emit.isWholeWord();
-    }
-
     @Override
     public boolean isMatch() {
         return true;

src/main/java/org/ahocorasick/trie/State.java

@@ -43,7 +43,7 @@ public class State {
     private State failure = null;
 
     /** whenever this state is reached, it will emit the matches keywords for future reference */
-    private Set<String> emits = null;
+    private Set<Keyword> emits = null;
 
     public State() {
         this(0);
@@ -83,21 +83,25 @@ public class State {
         return this.depth;
     }
 
-    public void addEmit(String keyword) {
+    public void addEmit(Keyword keyword) {
         if (this.emits == null) {
             this.emits = new TreeSet<>();
         }
         this.emits.add(keyword);
     }
 
-    public void addEmit(Collection<String> emits) {
+    public void addEmit(Collection<Keyword> emits) {
-        for (String emit : emits) {
+        for (Keyword emit : emits) {
             addEmit(emit);
         }
     }
     
-    public Collection<String> emit() {
+    public void addEmitString(String key) {
-        return this.emits == null ? Collections.<String> emptyList() : this.emits;
+        addEmit(new Keyword(key, depth));
+    }
+
+    public Collection<Keyword> emit() {
+        return this.emits == null ? Collections.<Keyword> emptyList() : this.emits;
     }
 
     public State failure(EmitCandidateFlushHandler emitCandidateFlushHandler) {

src/main/java/org/ahocorasick/trie/Token.java

@@ -2,7 +2,7 @@ package org.ahocorasick.trie;
 
 public abstract class Token {
 
-    private String fragment;
+    private final String fragment;
 
     public Token(String fragment) {
         this.fragment = fragment;
@@ -14,14 +14,6 @@ public abstract class Token {
 
     public abstract boolean isMatch();
 
-    public boolean isWholeWord() {
-        return false;
-    }
-
-    public boolean isWhiteSpace() {
-        return false;
-    }
-
     public abstract Emit getEmit();
 
 }

src/main/java/org/ahocorasick/trie/Transition.java

@@ -23,7 +23,7 @@ import java.util.Objects;
  * @author doug.lovell
  * @param <T>
  */
-public abstract class Transition<T> {
+public class Transition<T> {
     protected final T token;
     protected final int start;
     protected final int length;
@@ -46,11 +46,10 @@ public abstract class Transition<T> {
         return length;
     }
     
-    public abstract boolean isWordSeparator();
-    
     @Override
     public String toString() {
-        return "Transition on " + token;
+        return "Transition on '" + token + "' start: " + start +
+                ", length: " + length;
     }
     
     @Override

src/main/java/org/ahocorasick/trie/Trie.java

@@ -10,6 +10,8 @@ import org.ahocorasick.trie.handler.FirstMatchHandler;
 
 import java.util.Collection;
 import java.util.Queue;
+import java.util.LinkedList;
+import java.util.ListIterator;
 import java.util.concurrent.LinkedBlockingDeque;
 
 /**
@@ -43,34 +45,16 @@ public class Trie {
         public int getPosition() {
             return position;
         }
+        public abstract Emit match(Keyword kwd, int start, int position);
     }
     
     private class WordTokenizer extends KeywordTokenizer {
         public WordTokenizer(CharSequence input) {
             super(input);
-            System.out.println("WORDTOKENIZER input '" + input + "'");
-            // leading and trailing white space cannot be part of the
-            // search pattern
-            int start = 0;
-            while (start < length && Character.isWhitespace(input.charAt(start))) {
-                ++start;
-            }
-            int end = length - 1;
-            while (start < end && Character.isWhitespace(input.charAt(end))) {
-                --end;
-            }
-            this.input = input.subSequence(start, end + 1);
-            this.length = end - start + 1;
-            System.out.println("WORDTOKENIZER input '" + this.input + "'");
-            System.out.println("input.length " + this.input.length() + ", this.length " + this.length);
         }
         @Override
         public Transition<String> nextTransition() {
             WordTransition t = null;
-            System.out.println("WORDTOKENIZER get next word transition");
-            System.out.println("Position: " + position);
-            System.out.println("Text under cursor, '" + 
-                input.subSequence(Math.min(length-1, position), Math.min(length, position + 10)) + "'");
             while (position < length && Character.isWhitespace(currentChar())) {
                 ++position;
             }
@@ -82,10 +66,17 @@ public class Trie {
                 String word = input.subSequence(start, position).toString();
                 t = new WordTransition(word, start);
             }
-            System.out.println("New position: " + position);
-            System.out.println("Text in transition, '" + (t == null ? "null" : t.transitionToken()) + "'");
             return t;
         }
+        /*
+        On word matching, we return the matched text, which can be of different
+        length that the keyword, due to whitespace differences.
+        */
+        @Override
+        public Emit match(Keyword kwd, int start, int position) {
+            String matchedText = input.subSequence(start, position).toString();
+            return new Emit(start, position - 1, matchedText);
+        }
     }
     
     private class CharacterTokenizer extends KeywordTokenizer {
@@ -95,17 +86,20 @@ public class Trie {
         @Override
         public Transition<Character> nextTransition() {
             CharacterTransition t = null;
-            System.out.println("CHARACTERTOKENIZER get next character transition");
-            System.out.println("Position: " + position);
-            System.out.println("Text under cursor, '" + input.subSequence(position, Math.min(length, position + 10)) + "'");
             if (position < length) {
                 t = new CharacterTransition(currentChar(), position);
                 position += 1;
             }
-            System.out.println("New position: " + position);
-            System.out.println("Text in transition, '" + (t == null ? "null" : t.transitionToken()) + "'");
             return t;
         }
+        /*
+        On character matching, the tests expect the implementation to
+        return the matched keyword.
+        */
+        @Override
+        public Emit match(Keyword kwd, int start, int position) {
+            return new Emit(start, position - 1, kwd.getText());
+        }
     }
     
     private class TokenStream {
@@ -120,7 +114,7 @@ public class Trie {
                 input.append(trieConfig.isCaseInsensitive() ?
                         Character.toLowerCase(ch) : ch);
             }
-            if (trieConfig.hasWordTransitions()) {
+            if (trieConfig.isOnlyWholeWords()) {
                 kwt = new WordTokenizer(input);
             }
             else {
@@ -140,18 +134,14 @@ public class Trie {
             return next;
         }
         
-        public boolean isWholeWord(int start) {
-            if (lookahead == null) {
-                lookahead = kwt.nextTransition();
-            }
-            return ((start == 0 || 
-                     Character.isWhitespace(input.charAt(start-1))) && 
-                    (lookahead == null || lookahead.isWordSeparator()));
-        }
-        
         public String input() {
             return input.toString();
         }
+        
+        public Emit match(Keyword kwd, int start, int position) {
+            return kwt.match(kwd, start, position);
+        }
+
     }
         
     private void addKeyword(CharSequence keyword) {
@@ -165,7 +155,7 @@ public class Trie {
             currentState = currentState.addState(tn);
             tn = tknz.nextTransition();
         }
-        currentState.addEmit(tknz.input());
+        currentState.addEmitString(tknz.input());
     }
 
     public Collection<Token> tokenize(String text) {
@@ -201,27 +191,25 @@ public class Trie {
 
         TokenStream tknz = new TokenStream(text);
         
+        LinkedList<Transition> tknHistory = new LinkedList<>();
         State currentState = this.rootState;
         Transition tn = tknz.nextTransition();
         while (tn != null) {
             if (flushHandler.stop()) {
                 return;
             }
+            tknHistory.add(tn);
             currentState = getState(currentState, tn, flushHandler);
-            Collection<String> emits = currentState.emit();
+            Collection<Keyword> emits = currentState.emit();
-            for (String emit : emits) {
+            int depth = currentState.getDepth();
+            while (depth < tknHistory.size()) {
+                tknHistory.remove();
+            }
+            for (Keyword emit : emits) {
                 int position = tn.getStart() + tn.getLength();
-                int start = position - emit.length();
+                int start = tknHistory.get(depth - emit.getDepth()).getStart();
-                if (start < 0) {
+                ListIterator<Transition> tns = tknHistory.listIterator();
-                    System.out.println("START < 0 !! at " + position + " on '" + emit + "', length " + emit.length());
+                emitCandidateHolder.addCandidate(tknz.match(emit, start, position));
-                    System.out.println("TEXT is '" + text + "'");
-                    System.out.println(tn);
-                }
-                boolean isWholeWord = tknz.isWholeWord(start);
-                if (isWholeWord || !trieConfig.isOnlyWholeWords()) {
-                    emitCandidateHolder.addCandidate(
-                            new Emit(start, position - 1, emit, isWholeWord));
-                }
             }
             tn = tknz.nextTransition();
         }
@@ -290,16 +278,11 @@ public class Trie {
         }
 
         public TrieBuilder onlyWholeWords() {
-            this.trieConfig.setOnlyWholeWords(true);
-            return this;
-        }
-
-        public TrieBuilder wordTransitions() {
             if (hasAddedKeyword) {
                 throw new IllegalStateException(
-                    "Unable to switch to word transitions after keywords added");
+                    "Unable to switch to only whole words after keywords added");
             }
-            this.trieConfig.setWordTransitions(true);
+            this.trieConfig.setOnlyWholeWords(true);
             return this;
         }
 

src/main/java/org/ahocorasick/trie/TrieConfig.java

@@ -8,8 +8,6 @@ public class TrieConfig {
 
     private boolean caseInsensitive = false;
     
-    private boolean wordTransitions = false;
-
     public boolean isAllowOverlaps() {
         return allowOverlaps;
     }
@@ -33,12 +31,4 @@ public class TrieConfig {
     public void setCaseInsensitive(boolean caseInsensitive) {
         this.caseInsensitive = caseInsensitive;
     }
-    
-    public boolean hasWordTransitions() {
-        return wordTransitions;
-    }
-    
-    public void setWordTransitions(boolean wordNodes) {
-        this.wordTransitions = wordNodes;
-    }
 }

src/main/java/org/ahocorasick/trie/WordTransition.java

@@ -28,9 +28,4 @@ public class WordTransition extends Transition<String> {
     public WordTransition(String s) {
         this(s, 0);
     }
-
-    @Override
-    public boolean isWordSeparator() {
-        return true;
-    }
 }

src/test/java/org/ahocorasick/trie/TrieTest.java

@@ -81,7 +81,7 @@ public class TrieTest {
     @Test
     public void variousKeywordsFirstMatchWordTransitions() {
         Trie trie = Trie.builder()
-                .wordTransitions()
+                .onlyWholeWords()
                 .addKeyword("abc")
                 .addKeyword("bcd")
                 .addKeyword("cde")
@@ -198,7 +198,7 @@ public class TrieTest {
 @Test
     public void recipesWordTransitions() {
         Trie trie = Trie.builder()
-                .wordTransitions()
+                .onlyWholeWords()
                 .addKeyword("veal")
                 .addKeyword("cauliflower")
                 .addKeyword("broccoli")
@@ -265,7 +265,7 @@ public class TrieTest {
     public void nonOverlappingWordTransitions() {
         Trie trie = Trie.builder()
                 .removeOverlaps()
-                .wordTransitions()
+                .onlyWholeWords()
                 .addKeyword("peper molen")
                 .addKeyword("molen wiel")
                 .addKeyword("wiel dop")
@@ -436,7 +436,7 @@ public class TrieTest {
     @Test
     public void tokenizeFullSentenceByWords() {
         Trie trie = Trie.builder()
-                .wordTransitions()
+                .onlyWholeWords()
                 .addKeyword("Alpha")
                 .addKeyword("Beta")
                 .addKeyword("Gamma")
@@ -457,13 +457,13 @@ public class TrieTest {
     public ExpectedException thrown = ExpectedException.none();
     
     @Test
-    public void wordTransitionsThrowsExceptionAfterKeywordsAdded()
+    public void onlyWholeWordsThrowsExceptionAfterKeywordsAdded()
       throws IllegalStateException {
         thrown.expect(IllegalStateException.class);
-        thrown.expectMessage("Unable to switch to word transitions after keywords added");
+        thrown.expectMessage("Unable to switch to only whole words after keywords added");
         Trie trie = Trie.builder()
                 .addKeyword("Happy for now")
-                .wordTransitions()
+                .onlyWholeWords()
                 .addKeyword("Not so happy")
                 .build();
     }
@@ -590,45 +590,26 @@ public class TrieTest {
     }
 
     /*
-    What does "onlyWholeWords" mean when the keyword itself has spaces?
+    For onlyWholeWords, we'll ignore leading and trailing white space
-    @Test
-    public void spacesAroundKeyword() {
-        String keyword = " lorem ipso facto genera linden pharma six 1 ";
-        Trie trie = Trie.builder()
-                .onlyWholeWords()
-                .caseInsensitive()
-                .addKeyword(keyword)
-                .build();
-        Collection < Emit > emits = trie.parseText(
-                "gravita conundrum" + keyword + "under addressed object ");
-        assertEquals(1, emits.size());
-        checkEmit(emits.iterator().next(), 0, keyword.length() + 1, keyword);
-    }
-    */
-    
-    /*
-    For wordTransitions, we'll ignore leading and trailing white space
     included on keywords
     */
     @Test
     public void spacesAroundKeywordByWords() {
         String keyword = "lorem ipso facto genera linden pharma six 1";
         Trie trie = Trie.builder()
-                .wordTransitions()
+                .onlyWholeWords()
                 .caseInsensitive()
                 .addKeyword(" " + keyword + " ")
                 .build();
         Collection < Emit > emits = trie.parseText(
                 keyword + " under addressed object ");
         assertEquals(1, emits.size());
-        checkEmit(emits.iterator().next(), 0, keyword.length(), keyword);
+        checkEmit(emits.iterator().next(), 0, keyword.length() - 1, keyword);
     }
 
     private void assertToken(Token token, String fragment, boolean match, boolean wholeWord, boolean whiteSpace) {
         assertEquals(fragment, token.getFragment());
         assertEquals(match, token.isMatch());
-        assertEquals(wholeWord, token.isWholeWord());
-        assertEquals(whiteSpace, token.isWhiteSpace());
     }
 
     private void checkEmit(Emit next, int expectedStart, int expectedEnd, String expectedKeyword) {