Issue #2 implemented whole word matching

This commit is contained in:
robert-bor 2014-02-01 20:35:38 +01:00
parent 47383f1dbc
commit cb44a6bff2
4 changed files with 59 additions and 3 deletions

View File

@ -79,6 +79,19 @@ conflict resolution rules: 1) longer matches prevail over shorter matches, 2) le
There is only one result now:
* "hot chocolate" starting at position 0, ending at position 12
If you want the algorithm to only check for whole words, you can tell the Trie to do so:
```java
Trie trie = new Trie().onlyWholeWords();
trie.addKeyword("sugar");
Collection<Emit> emits = trie.parseText("sugarcane sugarcane sugar canesugar");
```
In this case, it will only find one match, whereas it would normally find four. The sugarcane/canesugar words
are discarded because they are partial matches.
License
-------
Licensed under the Apache License, Version 2.0 (the "License");

View File

@ -36,6 +36,11 @@ public class Trie {
return this;
}
public Trie onlyWholeWords() {
this.trieConfig.setOnlyWholeWords(true);
return this;
}
public void addKeyword(String keyword) {
State currentState = this.rootState;
@ -58,6 +63,10 @@ public class Trie {
position++;
}
if (trieConfig.isOnlyWholeWords()) {
removePartialMatches(text, collectedEmits);
}
if (!trieConfig.isAllowOverlaps()) {
IntervalTree intervalTree = new IntervalTree((List<Intervalable>)(List<?>)collectedEmits);
intervalTree.removeOverlaps((List<Intervalable>) (List<?>) collectedEmits);
@ -66,6 +75,24 @@ public class Trie {
return collectedEmits;
}
private void removePartialMatches(String searchText, List<Emit> collectedEmits) {
long size = searchText.length();
List<Emit> removeEmits = new ArrayList<Emit>();
for (Emit emit : collectedEmits) {
if ((emit.getStart() == 0 ||
searchText.charAt(emit.getStart() - 1) == ' ') &&
(emit.getEnd() == size ||
searchText.charAt(emit.getEnd() + 1) == ' ')) {
continue;
}
removeEmits.add(emit);
}
for (Emit removeEmit : removeEmits) {
collectedEmits.remove(removeEmit);
}
}
private State getState(State currentState, Character character) {
State newCurrentState = currentState.nextState(character);
while (newCurrentState == null) {

View File

@ -4,6 +4,8 @@ public class TrieConfig {
private boolean allowOverlaps = true;
private boolean onlyWholeWords = false;
public boolean isAllowOverlaps() {
return allowOverlaps;
}
@ -12,4 +14,12 @@ public class TrieConfig {
this.allowOverlaps = allowOverlaps;
}
public boolean isOnlyWholeWords() {
return onlyWholeWords;
}
public void setOnlyWholeWords(boolean onlyWholeWords) {
this.onlyWholeWords = onlyWholeWords;
}
}

View File

@ -123,12 +123,18 @@ public class TrieTest {
trie.addKeyword("n");
trie.addKeyword("urning");
Collection<Emit> emits = trie.parseText("Turning");
for (Emit emit : emits) {
System.out.println(emit.getStart()+":"+emit.getEnd()+"="+emit.getKeyword());
}
assertEquals(2, emits.size());
}
@Test
public void partialMatch() {
Trie trie = new Trie().onlyWholeWords();
trie.addKeyword("sugar");
Collection<Emit> emits = trie.parseText("sugarcane sugarcane sugar canesugar"); // left, middle, right test
assertEquals(1, emits.size()); // Match must not be made
checkEmit(emits.iterator().next(), 20, 24, "sugar");
}
private void checkEmit(Emit next, int expectedStart, int expectedEnd, String expectedKeyword) {
assertEquals(expectedStart, next.getStart());
assertEquals(expectedEnd, next.getEnd());