diff --git a/README.md b/README.md index b32c965..843a336 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,17 @@ Aho-Corasick ============ +Dependency +---------- +Include this dependency in your POM. Be sure to check for the latest version in Maven Central. +```xml + + org.ahocorasick + ahocorasick + 0.1.0 + +``` + Introduction ------------ Nowadays most free-text searching is based on Lucene-like approaches, where the search text is parsed into its @@ -49,11 +60,9 @@ Setting up the Trie is a piece of cake: ``` You can now read the set. In this case it will find the following: -* "she" at position 3 -* "he" at position 3 -* "hers" at position 5 - -Note that the end-positions of the keyword match are emitted, not the start-positions! +* "she" starting at position 1, ending at position 3 +* "he" starting at position 2, ending at position 3 +* "hers" starting at position 2, ending at position 5 License ------- diff --git a/pom.xml b/pom.xml index 72f0bc2..0b0557f 100644 --- a/pom.xml +++ b/pom.xml @@ -1,8 +1,8 @@ 4.0.0 - org.aho-corasick - efficient-string-matching + org.ahocorasick + ahocorasick 0.2.0-SNAPSHOT jar Aho-CoraSick algorithm for efficient string matching diff --git a/src/main/java/org/ahocorasick/Emit.java b/src/main/java/org/ahocorasick/Emit.java deleted file mode 100644 index dfc96fb..0000000 --- a/src/main/java/org/ahocorasick/Emit.java +++ /dev/null @@ -1,26 +0,0 @@ -package org.ahocorasick; - -public class Emit { - private final int start; - private final int end; - private final String keyword; - - public Emit(final int start, final int end, final String keyword) { - this.start = start; - this.end = end; - this.keyword = keyword; - } - - public int getStart() { - return this.start; - } - - public int getEnd() { - return this.end; - } - - public String getKeyword() { - return this.keyword; - } - -} diff --git a/src/main/java/org/ahocorasick/interval/Interval.java b/src/main/java/org/ahocorasick/interval/Interval.java new file mode 100644 index 0000000..b457500 --- /dev/null +++ b/src/main/java/org/ahocorasick/interval/Interval.java @@ -0,0 +1,57 @@ +package org.ahocorasick.interval; + +public class Interval implements Intervalable { + + private int start; + private int end; + + public Interval(final int start, final int end) { + this.start = start; + this.end = end; + } + + public int getStart() { + return this.start; + } + + public int getEnd() { + return this.end; + } + + public int size() { + return end - start + 1; + } + + public boolean overlapsWith(Interval other) { + return this.start <= other.getEnd() && + this.end >= other.getStart(); + } + + public boolean overlapsWith(int point) { + return this.start <= point && point <= this.end; + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof Intervalable)) { + return false; + } + Intervalable other = (Intervalable)o; + return this.start == other.getStart() && + this.end == other.getEnd(); + } + + @Override + public int hashCode() { + return this.start % 100 + this.end % 100; + } + + @Override + public int compareTo(Object o) { + if (!(o instanceof Intervalable)) { + return -1; + } + Intervalable other = (Intervalable)o; + return this.start - other.getStart(); + } +} diff --git a/src/main/java/org/ahocorasick/interval/IntervalNode.java b/src/main/java/org/ahocorasick/interval/IntervalNode.java new file mode 100644 index 0000000..11db0ae --- /dev/null +++ b/src/main/java/org/ahocorasick/interval/IntervalNode.java @@ -0,0 +1,119 @@ +package org.ahocorasick.interval; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public class IntervalNode { + + private enum Direction { LEFT, RIGHT } + + private IntervalNode left = null; + private IntervalNode right = null; + private int point; + private List intervals = new ArrayList(); + + public IntervalNode(List intervals) { + this.point = determineMedian(intervals); + + List toLeft = new ArrayList(); + List toRight = new ArrayList(); + + for (Intervalable interval : intervals) { + if (interval.getEnd() < this.point) { + toLeft.add(interval); + } else if (interval.getStart() > this.point) { + toRight.add(interval); + } else { + this.intervals.add(interval); + } + } + + if (toLeft.size() > 0) { + this.left = new IntervalNode(toLeft); + } + if (toRight.size() > 0) { + this.right = new IntervalNode(toRight); + } + } + + public int determineMedian(List intervals) { + int start = -1; + int end = -1; + for (Intervalable interval : intervals) { + int currentStart = interval.getStart(); + int currentEnd = interval.getEnd(); + if (start == -1 || currentStart < start) { + start = currentStart; + } + if (end == -1 || currentEnd > end) { + end = currentEnd; + } + } + return (start + end) / 2; + } + + public List findOverlaps(Intervalable interval) { + + List overlaps = new ArrayList(); + + if (this.point < interval.getStart()) { // Tends to the right + addToOverlaps(interval, overlaps, findOverlappingRanges(this.right, interval)); + addToOverlaps(interval, overlaps, checkForOverlapsToTheRight(interval)); + } else if (this.point > interval.getEnd()) { // Tends to the left + addToOverlaps(interval, overlaps, findOverlappingRanges(this.left, interval)); + addToOverlaps(interval, overlaps, checkForOverlapsToTheLeft(interval)); + } else { // Somewhere in the middle + addToOverlaps(interval, overlaps, this.intervals); + addToOverlaps(interval, overlaps, findOverlappingRanges(this.left, interval)); + addToOverlaps(interval, overlaps, findOverlappingRanges(this.right, interval)); + } + + return overlaps; + } + + protected void addToOverlaps(Intervalable interval, List overlaps, List newOverlaps) { + for (Intervalable currentInterval : newOverlaps) { + if (!currentInterval.equals(interval)) { + overlaps.add(currentInterval); + } + } + } + + protected List checkForOverlapsToTheLeft(Intervalable interval) { + return checkForOverlaps(interval, Direction.LEFT); + } + + protected List checkForOverlapsToTheRight(Intervalable interval) { + return checkForOverlaps(interval, Direction.RIGHT); + } + + protected List checkForOverlaps(Intervalable interval, Direction direction) { + + List overlaps = new ArrayList(); + for (Intervalable currentInterval : this.intervals) { + switch (direction) { + case LEFT : + if (currentInterval.getStart() <= interval.getEnd()) { + overlaps.add(currentInterval); + } + break; + case RIGHT : + if (currentInterval.getEnd() >= interval.getStart()) { + overlaps.add(currentInterval); + } + break; + } + } + return overlaps; + } + + + protected List findOverlappingRanges(IntervalNode node, Intervalable interval) { + if (node != null) { + return node.findOverlaps(interval); + } + return Collections.emptyList(); + } + +} diff --git a/src/main/java/org/ahocorasick/interval/IntervalTree.java b/src/main/java/org/ahocorasick/interval/IntervalTree.java new file mode 100644 index 0000000..40eeb0b --- /dev/null +++ b/src/main/java/org/ahocorasick/interval/IntervalTree.java @@ -0,0 +1,48 @@ +package org.ahocorasick.interval; + +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.TreeSet; + +public class IntervalTree { + + private IntervalNode rootNode = null; + + public IntervalTree(List intervals) { + this.rootNode = new IntervalNode(intervals); + } + + public List removeOverlaps(List intervals) { + + // Sort the intervals on size, then left-most position + Collections.sort(intervals, new IntervalableComparatorBySize()); + + Set removeIntervals = new TreeSet(); + + for (Intervalable interval : intervals) { + // If the interval was already removed, ignore it + if (removeIntervals.contains(interval)) { + continue; + } + + // Remove all overallping intervals + removeIntervals.addAll(findOverlaps(interval)); + } + + // Remove all intervals that were overlapping + for (Intervalable removeInterval : removeIntervals) { + intervals.remove(removeInterval); + } + + // Sort the intervals, now on left-most position only + Collections.sort(intervals, new IntervalableComparatorByPosition()); + + return intervals; + } + + public List findOverlaps(Intervalable interval) { + return rootNode.findOverlaps(interval); + } + +} diff --git a/src/main/java/org/ahocorasick/interval/Intervalable.java b/src/main/java/org/ahocorasick/interval/Intervalable.java new file mode 100644 index 0000000..286a232 --- /dev/null +++ b/src/main/java/org/ahocorasick/interval/Intervalable.java @@ -0,0 +1,9 @@ +package org.ahocorasick.interval; + +public interface Intervalable extends Comparable { + + public int getStart(); + public int getEnd(); + public int size(); + +} diff --git a/src/main/java/org/ahocorasick/interval/IntervalableComparatorByPosition.java b/src/main/java/org/ahocorasick/interval/IntervalableComparatorByPosition.java new file mode 100644 index 0000000..d144995 --- /dev/null +++ b/src/main/java/org/ahocorasick/interval/IntervalableComparatorByPosition.java @@ -0,0 +1,12 @@ +package org.ahocorasick.interval; + +import java.util.Comparator; + +public class IntervalableComparatorByPosition implements Comparator { + + @Override + public int compare(Intervalable intervalable, Intervalable intervalable2) { + return intervalable.getStart() - intervalable2.getStart(); + } + +} diff --git a/src/main/java/org/ahocorasick/interval/IntervalableComparatorBySize.java b/src/main/java/org/ahocorasick/interval/IntervalableComparatorBySize.java new file mode 100644 index 0000000..3814759 --- /dev/null +++ b/src/main/java/org/ahocorasick/interval/IntervalableComparatorBySize.java @@ -0,0 +1,16 @@ +package org.ahocorasick.interval; + +import java.util.Comparator; + +public class IntervalableComparatorBySize implements Comparator { + + @Override + public int compare(Intervalable intervalable, Intervalable intervalable2) { + int comparison = intervalable2.size() - intervalable.size(); + if (comparison == 0) { + comparison = intervalable.getStart() - intervalable2.getStart(); + } + return comparison; + } + +} diff --git a/src/main/java/org/ahocorasick/trie/Emit.java b/src/main/java/org/ahocorasick/trie/Emit.java new file mode 100644 index 0000000..d6644ef --- /dev/null +++ b/src/main/java/org/ahocorasick/trie/Emit.java @@ -0,0 +1,19 @@ +package org.ahocorasick.trie; + +import org.ahocorasick.interval.Interval; +import org.ahocorasick.interval.Intervalable; + +public class Emit extends Interval implements Intervalable { + + private final String keyword; + + public Emit(final int start, final int end, final String keyword) { + super(start, end); + this.keyword = keyword; + } + + public String getKeyword() { + return this.keyword; + } + +} diff --git a/src/main/java/org/ahocorasick/State.java b/src/main/java/org/ahocorasick/trie/State.java similarity index 99% rename from src/main/java/org/ahocorasick/State.java rename to src/main/java/org/ahocorasick/trie/State.java index 535f70d..9108838 100644 --- a/src/main/java/org/ahocorasick/State.java +++ b/src/main/java/org/ahocorasick/trie/State.java @@ -1,4 +1,4 @@ -package org.ahocorasick; +package org.ahocorasick.trie; import java.util.*; diff --git a/src/main/java/org/ahocorasick/Trie.java b/src/main/java/org/ahocorasick/trie/Trie.java similarity index 83% rename from src/main/java/org/ahocorasick/Trie.java rename to src/main/java/org/ahocorasick/trie/Trie.java index 16d8e88..769402a 100644 --- a/src/main/java/org/ahocorasick/Trie.java +++ b/src/main/java/org/ahocorasick/trie/Trie.java @@ -1,4 +1,7 @@ -package org.ahocorasick; +package org.ahocorasick.trie; + +import org.ahocorasick.interval.IntervalTree; +import org.ahocorasick.interval.Intervalable; import java.util.ArrayList; import java.util.Collection; @@ -13,14 +16,26 @@ import java.util.concurrent.LinkedBlockingDeque; */ public class Trie { + private TrieConfig trieConfig; + private State rootState; private boolean failureStatesConstructed = false; - public Trie() { + public Trie(TrieConfig trieConfig) { + this.trieConfig = trieConfig; this.rootState = new State(); } + public Trie() { + this(new TrieConfig()); + } + + public Trie removeOverlaps() { + this.trieConfig.setAllowOverlaps(false); + return this; + } + public void addKeyword(String keyword) { State currentState = this.rootState; @@ -30,6 +45,7 @@ public class Trie { currentState.addEmit(keyword); } + @SuppressWarnings("unchecked") public Collection parseText(String text) { checkForConstructedFailureStates(); @@ -41,6 +57,12 @@ public class Trie { storeEmits(position, currentState, collectedEmits); position++; } + + if (!trieConfig.isAllowOverlaps()) { + IntervalTree intervalTree = new IntervalTree((List)(List)collectedEmits); + intervalTree.removeOverlaps((List) (List) collectedEmits); + } + return collectedEmits; } diff --git a/src/main/java/org/ahocorasick/trie/TrieConfig.java b/src/main/java/org/ahocorasick/trie/TrieConfig.java new file mode 100644 index 0000000..23bed1f --- /dev/null +++ b/src/main/java/org/ahocorasick/trie/TrieConfig.java @@ -0,0 +1,15 @@ +package org.ahocorasick.trie; + +public class TrieConfig { + + private boolean allowOverlaps = true; + + public boolean isAllowOverlaps() { + return allowOverlaps; + } + + public void setAllowOverlaps(boolean allowOverlaps) { + this.allowOverlaps = allowOverlaps; + } + +} diff --git a/src/test/java/org/ahocorasick/interval/IntervalTest.java b/src/test/java/org/ahocorasick/interval/IntervalTest.java new file mode 100644 index 0000000..e61bad7 --- /dev/null +++ b/src/test/java/org/ahocorasick/interval/IntervalTest.java @@ -0,0 +1,57 @@ +package org.ahocorasick.interval; + +import org.junit.Test; + +import java.util.*; + +import static junit.framework.Assert.assertEquals; +import static junit.framework.Assert.assertFalse; +import static junit.framework.Assert.assertTrue; + +public class IntervalTest { + + @Test + public void construct() { + Interval i = new Interval(1,3); + assertEquals(1, i.getStart()); + assertEquals(3, i.getEnd()); + } + + @Test + public void size() { + assertEquals(3, new Interval(0,2).size()); + } + + @Test + public void intervaloverlaps() { + assertTrue(new Interval(1,3).overlapsWith(new Interval(2,4))); + } + + @Test + public void intervalDoesNotOverlap() { + assertFalse(new Interval(1, 13).overlapsWith(new Interval(27, 42))); + } + + @Test + public void pointOverlaps() { + assertTrue(new Interval(1,3).overlapsWith(2)); + } + + @Test + public void pointDoesNotOverlap() { + assertFalse(new Interval(1, 13).overlapsWith(42)); + } + + @Test + public void comparable() { + Set intervals = new TreeSet(); + intervals.add(new Interval(4, 6)); + intervals.add(new Interval(2, 7)); + intervals.add(new Interval(3, 4)); + Iterator it = intervals.iterator(); + assertEquals(2, it.next().getStart()); + assertEquals(3, it.next().getStart()); + assertEquals(4, it.next().getStart()); + } + +} diff --git a/src/test/java/org/ahocorasick/interval/IntervalTreeTest.java b/src/test/java/org/ahocorasick/interval/IntervalTreeTest.java new file mode 100644 index 0000000..f4a7f57 --- /dev/null +++ b/src/test/java/org/ahocorasick/interval/IntervalTreeTest.java @@ -0,0 +1,51 @@ +package org.ahocorasick.interval; + +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +import static junit.framework.Assert.assertEquals; + +public class IntervalTreeTest { + + @Test + public void findOverlaps() { + List intervals = new ArrayList(); + intervals.add(new Interval(0, 2)); + intervals.add(new Interval(1, 3)); + intervals.add(new Interval(2, 4)); + intervals.add(new Interval(3, 5)); + intervals.add(new Interval(4, 6)); + intervals.add(new Interval(5, 7)); + IntervalTree intervalTree = new IntervalTree(intervals); + List overlaps = intervalTree.findOverlaps(new Interval(1,3)); + assertEquals(3, overlaps.size()); + Iterator overlapsIt = overlaps.iterator(); + assertOverlap(overlapsIt.next(), 2, 4); + assertOverlap(overlapsIt.next(), 3, 5); + assertOverlap(overlapsIt.next(), 0, 2); + } + + @Test + public void removeOverlaps() { + List intervals = new ArrayList(); + intervals.add(new Interval(0, 2)); + intervals.add(new Interval(4, 5)); + intervals.add(new Interval(2, 10)); + intervals.add(new Interval(6, 13)); + intervals.add(new Interval(9, 15)); + intervals.add(new Interval(12, 16)); + IntervalTree intervalTree = new IntervalTree(intervals); + intervals = intervalTree.removeOverlaps(intervals); + assertEquals(2, intervals.size()); + + } + + protected void assertOverlap(Intervalable interval, int expectedStart, int expectedEnd) { + assertEquals(expectedStart, interval.getStart()); + assertEquals(expectedEnd, interval.getEnd()); + } + +} diff --git a/src/test/java/org/ahocorasick/interval/IntervalableComparatorByPositionTest.java b/src/test/java/org/ahocorasick/interval/IntervalableComparatorByPositionTest.java new file mode 100644 index 0000000..a6f1017 --- /dev/null +++ b/src/test/java/org/ahocorasick/interval/IntervalableComparatorByPositionTest.java @@ -0,0 +1,25 @@ +package org.ahocorasick.interval; + +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static junit.framework.Assert.assertEquals; + +public class IntervalableComparatorByPositionTest { + + @Test + public void sortOnPosition() { + List intervals = new ArrayList(); + intervals.add(new Interval(4,5)); + intervals.add(new Interval(1,4)); + intervals.add(new Interval(3,8)); + Collections.sort(intervals, new IntervalableComparatorByPosition()); + assertEquals(4, intervals.get(0).size()); + assertEquals(6, intervals.get(1).size()); + assertEquals(2, intervals.get(2).size()); + } + +} diff --git a/src/test/java/org/ahocorasick/interval/IntervalableComparatorBySizeTest.java b/src/test/java/org/ahocorasick/interval/IntervalableComparatorBySizeTest.java new file mode 100644 index 0000000..208cf3d --- /dev/null +++ b/src/test/java/org/ahocorasick/interval/IntervalableComparatorBySizeTest.java @@ -0,0 +1,35 @@ +package org.ahocorasick.interval; + +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static junit.framework.Assert.assertEquals; + +public class IntervalableComparatorBySizeTest { + + @Test + public void sortOnSize() { + List intervals = new ArrayList(); + intervals.add(new Interval(4,5)); + intervals.add(new Interval(1,4)); + intervals.add(new Interval(3,8)); + Collections.sort(intervals, new IntervalableComparatorBySize()); + assertEquals(6, intervals.get(0).size()); + assertEquals(4, intervals.get(1).size()); + assertEquals(2, intervals.get(2).size()); + } + + @Test + public void sortOnSizeThenPosition() { + List intervals = new ArrayList(); + intervals.add(new Interval(4,7)); + intervals.add(new Interval(2,5)); + Collections.sort(intervals, new IntervalableComparatorBySize()); + assertEquals(2, intervals.get(0).getStart()); + assertEquals(4, intervals.get(1).getStart()); + } + +} diff --git a/src/test/java/org/ahocorasick/trie/EmitTest.java b/src/test/java/org/ahocorasick/trie/EmitTest.java new file mode 100644 index 0000000..248fcc9 --- /dev/null +++ b/src/test/java/org/ahocorasick/trie/EmitTest.java @@ -0,0 +1,24 @@ +package org.ahocorasick.trie; + +import org.junit.Test; + +import static junit.framework.Assert.assertEquals; +import static junit.framework.Assert.assertNotSame; + +public class EmitTest { + + @Test + public void equals() { + Emit one = new Emit(13, 42, null); + Emit two = new Emit(13, 42, null); + assertEquals(one, two); + } + + @Test + public void notEquals() { + Emit one = new Emit(13, 42, null); + Emit two = new Emit(13, 43, null); + assertNotSame(one, two); + } + +} diff --git a/src/test/java/org/ahocorasick/StateTest.java b/src/test/java/org/ahocorasick/trie/StateTest.java similarity index 90% rename from src/test/java/org/ahocorasick/StateTest.java rename to src/test/java/org/ahocorasick/trie/StateTest.java index 96139b1..2a64370 100644 --- a/src/test/java/org/ahocorasick/StateTest.java +++ b/src/test/java/org/ahocorasick/trie/StateTest.java @@ -1,5 +1,6 @@ -package org.ahocorasick; +package org.ahocorasick.trie; +import org.ahocorasick.trie.State; import org.junit.Test; import static junit.framework.Assert.assertEquals; diff --git a/src/test/java/org/ahocorasick/TrieTest.java b/src/test/java/org/ahocorasick/trie/TrieTest.java similarity index 69% rename from src/test/java/org/ahocorasick/TrieTest.java rename to src/test/java/org/ahocorasick/trie/TrieTest.java index ac16471..17019f9 100644 --- a/src/test/java/org/ahocorasick/TrieTest.java +++ b/src/test/java/org/ahocorasick/trie/TrieTest.java @@ -1,5 +1,7 @@ -package org.ahocorasick; +package org.ahocorasick.trie; +import org.ahocorasick.trie.Emit; +import org.ahocorasick.trie.Trie; import org.junit.Test; import java.util.Collection; @@ -77,6 +79,35 @@ public class TrieTest { checkEmit(iterator.next(), 51, 58, "broccoli"); } + @Test + public void longAndShortOverlappingMatch() { + Trie trie = new Trie(); + trie.addKeyword("he"); + trie.addKeyword("hehehehe"); + Collection emits = trie.parseText("hehehehehe"); + Iterator iterator = emits.iterator(); + checkEmit(iterator.next(), 0, 1, "he"); + checkEmit(iterator.next(), 2, 3, "he"); + checkEmit(iterator.next(), 4, 5, "he"); + checkEmit(iterator.next(), 0, 7, "hehehehe"); + checkEmit(iterator.next(), 6, 7, "he"); + checkEmit(iterator.next(), 2, 9, "hehehehe"); + checkEmit(iterator.next(), 8, 9, "he"); + } + + @Test + public void nonOverlapping() { + Trie trie = new Trie().removeOverlaps(); + trie.addKeyword("ab"); + trie.addKeyword("cba"); + trie.addKeyword("ababc"); + Collection emits = trie.parseText("ababcbab"); + Iterator iterator = emits.iterator(); + // With overlaps: ab@1, ab@3, ababc@4, cba@6, ab@7 + checkEmit(iterator.next(), 0, 4, "ababc"); + checkEmit(iterator.next(), 6, 7, "ab"); + } + private void checkEmit(Emit next, int expectedStart, int expectedEnd, String expectedKeyword) { assertEquals(expectedStart, next.getStart()); assertEquals(expectedEnd, next.getEnd());