Marmelator/aho-corasick
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Solved issue #5 by introducing a proper boundary check for words that are at the end of a String

Browse Source
This commit is contained in:
robert-bor 2014-02-08 12:45:55 +01:00
parent 1656e862df
commit 31117d6a6e
3 changed files with 34 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
pom.xml
View File

@ -3,7 +3,7 @@
<groupId>org.ahocorasick</groupId>
<artifactId>ahocorasick</artifactId>
<version>0.2.0</version>
<version>0.3.0-SNAPSHOT</version>
<packaging>jar</packaging>
<name>Aho-CoraSick algorithm for efficient string matching</name>
<description>Java library for efficient string matching against a large set of keywords</description>

2
src/main/java/org/ahocorasick/trie/Trie.java
View File

@ -117,7 +117,7 @@ public class Trie {
for (Emit emit : collectedEmits) {
if ((emit.getStart() == 0 ||
!Character.isAlphabetic(searchText.charAt(emit.getStart() - 1))) &&
(emit.getEnd() == size ||
(emit.getEnd() + 1 == size ||
!Character.isAlphabetic(searchText.charAt(emit.getEnd() + 1)))) {
continue;
}

50
src/test/java/org/ahocorasick/trie/TrieTest.java
View File

@ -1,7 +1,5 @@
package org.ahocorasick.trie;
import org.ahocorasick.trie.Emit;
import org.ahocorasick.trie.Trie;
import org.junit.Test;
import java.util.Collection;
@ -135,22 +133,6 @@ public class TrieTest {
checkEmit(emits.iterator().next(), 20, 24, "sugar");
}
@Test
public void caseInsensitive() {
Trie trie = new Trie().caseInsensitive();
trie.addKeyword("turning");
trie.addKeyword("once");
trie.addKeyword("again");
trie.addKeyword("börkü");
Collection<Emit> emits = trie.parseText("TurninG OnCe AgAiN BÖRKÜ");
assertEquals(4, emits.size()); // Match must not be made
Iterator<Emit> it = emits.iterator();
checkEmit(it.next(), 0, 6, "turning");
checkEmit(it.next(), 8, 11, "once");
checkEmit(it.next(), 13, 17, "again");
checkEmit(it.next(), 19, 23, "börkü");
}
@Test
public void tokenizeFullSentence() {
Trie trie = new Trie();
@ -169,6 +151,38 @@ public class TrieTest {
assertEquals(" in reserve", tokensIt.next().getFragment());
}
@Test
public void bug5InGithubReportedByXCurry() {
Trie trie = new Trie().caseInsensitive().onlyWholeWords();
trie.addKeyword("turning");
trie.addKeyword("once");
trie.addKeyword("again");
trie.addKeyword("börkü");
Collection<Emit> emits = trie.parseText("TurninG OnCe AgAiN BÖRKÜ");
assertEquals(4, emits.size()); // Match must not be made
Iterator<Emit> it = emits.iterator();
checkEmit(it.next(), 0, 6, "turning");
checkEmit(it.next(), 8, 11, "once");
checkEmit(it.next(), 13, 17, "again");
checkEmit(it.next(), 19, 23, "börkü");
}
@Test
public void caseInsensitive() {
Trie trie = new Trie().caseInsensitive();
trie.addKeyword("turning");
trie.addKeyword("once");
trie.addKeyword("again");
trie.addKeyword("börkü");
Collection<Emit> emits = trie.parseText("TurninG OnCe AgAiN BÖRKÜ");
assertEquals(4, emits.size()); // Match must not be made
Iterator<Emit> it = emits.iterator();
checkEmit(it.next(), 0, 6, "turning");
checkEmit(it.next(), 8, 11, "once");
checkEmit(it.next(), 13, 17, "again");
checkEmit(it.next(), 19, 23, "börkü");
}
@Test
public void tokenizeTokensInSequence() {
Trie trie = new Trie();