Solved issue #5 by introducing a proper boundary check for words that are at the end of a String
This commit is contained in:
parent
1656e862df
commit
31117d6a6e
2
pom.xml
2
pom.xml
@ -3,7 +3,7 @@
|
|||||||
|
|
||||||
<groupId>org.ahocorasick</groupId>
|
<groupId>org.ahocorasick</groupId>
|
||||||
<artifactId>ahocorasick</artifactId>
|
<artifactId>ahocorasick</artifactId>
|
||||||
<version>0.2.0</version>
|
<version>0.3.0-SNAPSHOT</version>
|
||||||
<packaging>jar</packaging>
|
<packaging>jar</packaging>
|
||||||
<name>Aho-CoraSick algorithm for efficient string matching</name>
|
<name>Aho-CoraSick algorithm for efficient string matching</name>
|
||||||
<description>Java library for efficient string matching against a large set of keywords</description>
|
<description>Java library for efficient string matching against a large set of keywords</description>
|
||||||
|
|||||||
@ -117,7 +117,7 @@ public class Trie {
|
|||||||
for (Emit emit : collectedEmits) {
|
for (Emit emit : collectedEmits) {
|
||||||
if ((emit.getStart() == 0 ||
|
if ((emit.getStart() == 0 ||
|
||||||
!Character.isAlphabetic(searchText.charAt(emit.getStart() - 1))) &&
|
!Character.isAlphabetic(searchText.charAt(emit.getStart() - 1))) &&
|
||||||
(emit.getEnd() == size ||
|
(emit.getEnd() + 1 == size ||
|
||||||
!Character.isAlphabetic(searchText.charAt(emit.getEnd() + 1)))) {
|
!Character.isAlphabetic(searchText.charAt(emit.getEnd() + 1)))) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,7 +1,5 @@
|
|||||||
package org.ahocorasick.trie;
|
package org.ahocorasick.trie;
|
||||||
|
|
||||||
import org.ahocorasick.trie.Emit;
|
|
||||||
import org.ahocorasick.trie.Trie;
|
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
|
|
||||||
import java.util.Collection;
|
import java.util.Collection;
|
||||||
@ -135,22 +133,6 @@ public class TrieTest {
|
|||||||
checkEmit(emits.iterator().next(), 20, 24, "sugar");
|
checkEmit(emits.iterator().next(), 20, 24, "sugar");
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
|
||||||
public void caseInsensitive() {
|
|
||||||
Trie trie = new Trie().caseInsensitive();
|
|
||||||
trie.addKeyword("turning");
|
|
||||||
trie.addKeyword("once");
|
|
||||||
trie.addKeyword("again");
|
|
||||||
trie.addKeyword("börkü");
|
|
||||||
Collection<Emit> emits = trie.parseText("TurninG OnCe AgAiN BÖRKÜ");
|
|
||||||
assertEquals(4, emits.size()); // Match must not be made
|
|
||||||
Iterator<Emit> it = emits.iterator();
|
|
||||||
checkEmit(it.next(), 0, 6, "turning");
|
|
||||||
checkEmit(it.next(), 8, 11, "once");
|
|
||||||
checkEmit(it.next(), 13, 17, "again");
|
|
||||||
checkEmit(it.next(), 19, 23, "börkü");
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void tokenizeFullSentence() {
|
public void tokenizeFullSentence() {
|
||||||
Trie trie = new Trie();
|
Trie trie = new Trie();
|
||||||
@ -169,6 +151,38 @@ public class TrieTest {
|
|||||||
assertEquals(" in reserve", tokensIt.next().getFragment());
|
assertEquals(" in reserve", tokensIt.next().getFragment());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void bug5InGithubReportedByXCurry() {
|
||||||
|
Trie trie = new Trie().caseInsensitive().onlyWholeWords();
|
||||||
|
trie.addKeyword("turning");
|
||||||
|
trie.addKeyword("once");
|
||||||
|
trie.addKeyword("again");
|
||||||
|
trie.addKeyword("börkü");
|
||||||
|
Collection<Emit> emits = trie.parseText("TurninG OnCe AgAiN BÖRKÜ");
|
||||||
|
assertEquals(4, emits.size()); // Match must not be made
|
||||||
|
Iterator<Emit> it = emits.iterator();
|
||||||
|
checkEmit(it.next(), 0, 6, "turning");
|
||||||
|
checkEmit(it.next(), 8, 11, "once");
|
||||||
|
checkEmit(it.next(), 13, 17, "again");
|
||||||
|
checkEmit(it.next(), 19, 23, "börkü");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void caseInsensitive() {
|
||||||
|
Trie trie = new Trie().caseInsensitive();
|
||||||
|
trie.addKeyword("turning");
|
||||||
|
trie.addKeyword("once");
|
||||||
|
trie.addKeyword("again");
|
||||||
|
trie.addKeyword("börkü");
|
||||||
|
Collection<Emit> emits = trie.parseText("TurninG OnCe AgAiN BÖRKÜ");
|
||||||
|
assertEquals(4, emits.size()); // Match must not be made
|
||||||
|
Iterator<Emit> it = emits.iterator();
|
||||||
|
checkEmit(it.next(), 0, 6, "turning");
|
||||||
|
checkEmit(it.next(), 8, 11, "once");
|
||||||
|
checkEmit(it.next(), 13, 17, "again");
|
||||||
|
checkEmit(it.next(), 19, 23, "börkü");
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void tokenizeTokensInSequence() {
|
public void tokenizeTokensInSequence() {
|
||||||
Trie trie = new Trie();
|
Trie trie = new Trie();
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user