From 31117d6a6ed114c2036510bb9976d9d0012ee59a Mon Sep 17 00:00:00 2001 From: robert-bor Date: Sat, 8 Feb 2014 12:45:55 +0100 Subject: [PATCH] Solved issue #5 by introducing a proper boundary check for words that are at the end of a String --- pom.xml | 2 +- src/main/java/org/ahocorasick/trie/Trie.java | 2 +- .../java/org/ahocorasick/trie/TrieTest.java | 50 ++++++++++++------- 3 files changed, 34 insertions(+), 20 deletions(-) diff --git a/pom.xml b/pom.xml index b1ae7cc..a9456f9 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ org.ahocorasick ahocorasick - 0.2.0 + 0.3.0-SNAPSHOT jar Aho-CoraSick algorithm for efficient string matching Java library for efficient string matching against a large set of keywords diff --git a/src/main/java/org/ahocorasick/trie/Trie.java b/src/main/java/org/ahocorasick/trie/Trie.java index 0d28c9b..fba65f0 100644 --- a/src/main/java/org/ahocorasick/trie/Trie.java +++ b/src/main/java/org/ahocorasick/trie/Trie.java @@ -117,7 +117,7 @@ public class Trie { for (Emit emit : collectedEmits) { if ((emit.getStart() == 0 || !Character.isAlphabetic(searchText.charAt(emit.getStart() - 1))) && - (emit.getEnd() == size || + (emit.getEnd() + 1 == size || !Character.isAlphabetic(searchText.charAt(emit.getEnd() + 1)))) { continue; } diff --git a/src/test/java/org/ahocorasick/trie/TrieTest.java b/src/test/java/org/ahocorasick/trie/TrieTest.java index 8000d63..7203fb5 100644 --- a/src/test/java/org/ahocorasick/trie/TrieTest.java +++ b/src/test/java/org/ahocorasick/trie/TrieTest.java @@ -1,7 +1,5 @@ package org.ahocorasick.trie; -import org.ahocorasick.trie.Emit; -import org.ahocorasick.trie.Trie; import org.junit.Test; import java.util.Collection; @@ -135,22 +133,6 @@ public class TrieTest { checkEmit(emits.iterator().next(), 20, 24, "sugar"); } - @Test - public void caseInsensitive() { - Trie trie = new Trie().caseInsensitive(); - trie.addKeyword("turning"); - trie.addKeyword("once"); - trie.addKeyword("again"); - trie.addKeyword("börkü"); - Collection emits = trie.parseText("TurninG OnCe AgAiN BÖRKÜ"); - assertEquals(4, emits.size()); // Match must not be made - Iterator it = emits.iterator(); - checkEmit(it.next(), 0, 6, "turning"); - checkEmit(it.next(), 8, 11, "once"); - checkEmit(it.next(), 13, 17, "again"); - checkEmit(it.next(), 19, 23, "börkü"); - } - @Test public void tokenizeFullSentence() { Trie trie = new Trie(); @@ -169,6 +151,38 @@ public class TrieTest { assertEquals(" in reserve", tokensIt.next().getFragment()); } + @Test + public void bug5InGithubReportedByXCurry() { + Trie trie = new Trie().caseInsensitive().onlyWholeWords(); + trie.addKeyword("turning"); + trie.addKeyword("once"); + trie.addKeyword("again"); + trie.addKeyword("börkü"); + Collection emits = trie.parseText("TurninG OnCe AgAiN BÖRKÜ"); + assertEquals(4, emits.size()); // Match must not be made + Iterator it = emits.iterator(); + checkEmit(it.next(), 0, 6, "turning"); + checkEmit(it.next(), 8, 11, "once"); + checkEmit(it.next(), 13, 17, "again"); + checkEmit(it.next(), 19, 23, "börkü"); + } + + @Test + public void caseInsensitive() { + Trie trie = new Trie().caseInsensitive(); + trie.addKeyword("turning"); + trie.addKeyword("once"); + trie.addKeyword("again"); + trie.addKeyword("börkü"); + Collection emits = trie.parseText("TurninG OnCe AgAiN BÖRKÜ"); + assertEquals(4, emits.size()); // Match must not be made + Iterator it = emits.iterator(); + checkEmit(it.next(), 0, 6, "turning"); + checkEmit(it.next(), 8, 11, "once"); + checkEmit(it.next(), 13, 17, "again"); + checkEmit(it.next(), 19, 23, "börkü"); + } + @Test public void tokenizeTokensInSequence() { Trie trie = new Trie();