diff --git a/src/main/java/org/ahocorasick/trie/Trie.java b/src/main/java/org/ahocorasick/trie/Trie.java index 3267c02..9f6c5bb 100644 --- a/src/main/java/org/ahocorasick/trie/Trie.java +++ b/src/main/java/org/ahocorasick/trie/Trie.java @@ -59,9 +59,9 @@ public class Trie { } int start = position; if (start < length) { - while (position < length && !Character.isWhitespace(currentChar())) { + do { ++position; - } + } while (position < length && Character.isLetterOrDigit(currentChar())); String word = input.subSequence(start, position).toString(); t = new WordTransition(word, start); } diff --git a/src/test/java/org/ahocorasick/trie/TrieTest.java b/src/test/java/org/ahocorasick/trie/TrieTest.java index 43a8fef..c4dde0e 100644 --- a/src/test/java/org/ahocorasick/trie/TrieTest.java +++ b/src/test/java/org/ahocorasick/trie/TrieTest.java @@ -2,6 +2,7 @@ package org.ahocorasick.trie; import org.ahocorasick.trie.handler.EmitHandler; import org.ahocorasick.trie.handler.SimpleEmitHandler; +import org.junit.Assert; import org.junit.Test; import java.util.ArrayList; @@ -608,6 +609,39 @@ public class TrieTest { checkEmit(emits.iterator().next(), 0, text.length() - 1, keyword); } + @Test + public void punctuationInText() { + Trie trie = Trie.builder() + .onlyWholeWords() + .addKeyword("pie") + .build(); + + Collection emits = trie.parseText("Want some pie? Gimme pie! pie, pie. The pie's revenge."); + Assert.assertEquals(5, emits.size()); + Iterator it = emits.iterator(); + checkEmit(it.next(), 10, 12, "pie"); + checkEmit(it.next(), 21, 23, "pie"); + checkEmit(it.next(), 26, 28, "pie"); + checkEmit(it.next(), 31, 33, "pie"); + checkEmit(it.next(), 40, 42, "pie"); + } + + @Test + public void punctuationInSearchTerm() { + Trie trie = Trie.builder() + .onlyWholeWords() + .addKeyword("Dr. Feelgood") + .addKeyword("Oi!") + .build(); + + Collection emits = trie + .parseText("The Oi! music genre is inspired by Dr. Feelgood and other bands. Oi or Dr Feelgood should not match."); + + Assert.assertEquals(2, emits.size()); + + + } + private void assertToken(Token token, String fragment, boolean match, boolean wholeWord, boolean whiteSpace) { assertEquals(fragment, token.getFragment()); assertEquals(match, token.isMatch());