Issue #1 remove overlapping intervals. Resolution rule: longer matches over smaller ones, left-most over right-most

README.md

@@ -1,6 +1,17 @@
 Aho-Corasick
 ============
 
+Dependency
+----------
+Include this dependency in your POM. Be sure to check for the latest version in Maven Central.
+```xml
+    <dependency>
+        <groupId>org.ahocorasick</groupId>
+        <artifactId>ahocorasick</artifactId>
+        <version>0.1.0</version>
+    </dependency>
+```
+
 Introduction
 ------------
 Nowadays most free-text searching is based on Lucene-like approaches, where the search text is parsed into its
@@ -49,11 +60,9 @@ Setting up the Trie is a piece of cake:
 ```
 
 You can now read the set. In this case it will find the following:
-* "she" at position 3
-* "he" at position 3
-* "hers" at position 5
-
-Note that the end-positions of the keyword match are emitted, not the start-positions!
+* "she" starting at position 1, ending at position 3
+* "he" starting at position 2, ending at position 3
+* "hers" starting at position 2, ending at position 5
 
 License
 -------

pom.xml

@@ -1,8 +1,8 @@
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <modelVersion>4.0.0</modelVersion>
 
-    <groupId>org.aho-corasick</groupId>
-    <artifactId>efficient-string-matching</artifactId>
+    <groupId>org.ahocorasick</groupId>
+    <artifactId>ahocorasick</artifactId>
     <version>0.2.0-SNAPSHOT</version>
     <packaging>jar</packaging>
     <name>Aho-CoraSick algorithm for efficient string matching</name>

src/main/java/org/ahocorasick/Emit.java

@@ -1,26 +0,0 @@
-package org.ahocorasick;
-
-public class Emit {
-    private final int start;
-    private final int end;
-    private final String keyword;
-
-    public Emit(final int start, final int end, final String keyword) {
-        this.start = start;
-        this.end = end;
-        this.keyword = keyword;
-    }
-
-    public int getStart() {
-        return this.start;
-    }
-
-    public int getEnd() {
-        return this.end;
-    }
-
-    public String getKeyword() {
-        return this.keyword;
-    }
-
-}

src/main/java/org/ahocorasick/interval/Interval.java

@@ -0,0 +1,57 @@
+package org.ahocorasick.interval;
+
+public class Interval implements Intervalable {
+
+    private int start;
+    private int end;
+
+    public Interval(final int start, final int end) {
+        this.start = start;
+        this.end = end;
+    }
+
+    public int getStart() {
+        return this.start;
+    }
+
+    public int getEnd() {
+        return this.end;
+    }
+
+    public int size() {
+        return end - start + 1;
+    }
+
+    public boolean overlapsWith(Interval other) {
+        return this.start <= other.getEnd() &&
+               this.end >= other.getStart();
+    }
+
+    public boolean overlapsWith(int point) {
+        return this.start <= point && point <= this.end;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (!(o instanceof Intervalable)) {
+            return false;
+        }
+        Intervalable other = (Intervalable)o;
+        return this.start == other.getStart() &&
+               this.end == other.getEnd();
+    }
+
+    @Override
+    public int hashCode() {
+        return this.start % 100 + this.end % 100;
+    }
+
+    @Override
+    public int compareTo(Object o) {
+        if (!(o instanceof Intervalable)) {
+            return -1;
+        }
+        Intervalable other = (Intervalable)o;
+        return this.start - other.getStart();
+    }
+}

src/main/java/org/ahocorasick/interval/IntervalNode.java

@@ -0,0 +1,119 @@
+package org.ahocorasick.interval;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+public class IntervalNode {
+
+    private enum Direction { LEFT, RIGHT }
+
+    private IntervalNode left = null;
+    private IntervalNode right = null;
+    private int point;
+    private List<Intervalable> intervals = new ArrayList<Intervalable>();
+
+    public IntervalNode(List<Intervalable> intervals) {
+        this.point = determineMedian(intervals);
+
+        List<Intervalable> toLeft = new ArrayList<Intervalable>();
+        List<Intervalable> toRight = new ArrayList<Intervalable>();
+
+        for (Intervalable interval : intervals) {
+            if (interval.getEnd() < this.point) {
+                toLeft.add(interval);
+            } else if (interval.getStart() > this.point) {
+                toRight.add(interval);
+            } else {
+                this.intervals.add(interval);
+            }
+        }
+
+        if (toLeft.size() > 0) {
+            this.left = new IntervalNode(toLeft);
+        }
+        if (toRight.size() > 0) {
+            this.right = new IntervalNode(toRight);
+        }
+    }
+
+    public int determineMedian(List<Intervalable> intervals) {
+        int start = -1;
+        int end = -1;
+        for (Intervalable interval : intervals) {
+            int currentStart = interval.getStart();
+            int currentEnd = interval.getEnd();
+            if (start == -1 || currentStart < start) {
+                start = currentStart;
+            }
+            if (end == -1 || currentEnd > end) {
+                end = currentEnd;
+            }
+        }
+        return (start + end) / 2;
+    }
+
+    public List<Intervalable> findOverlaps(Intervalable interval) {
+
+        List<Intervalable> overlaps = new ArrayList<Intervalable>();
+
+        if (this.point < interval.getStart()) { // Tends to the right
+            addToOverlaps(interval, overlaps, findOverlappingRanges(this.right, interval));
+            addToOverlaps(interval, overlaps, checkForOverlapsToTheRight(interval));
+        } else if (this.point > interval.getEnd()) { // Tends to the left
+            addToOverlaps(interval, overlaps, findOverlappingRanges(this.left, interval));
+            addToOverlaps(interval, overlaps, checkForOverlapsToTheLeft(interval));
+        } else { // Somewhere in the middle
+            addToOverlaps(interval, overlaps, this.intervals);
+            addToOverlaps(interval, overlaps, findOverlappingRanges(this.left, interval));
+            addToOverlaps(interval, overlaps, findOverlappingRanges(this.right, interval));
+        }
+
+        return overlaps;
+    }
+
+    protected void addToOverlaps(Intervalable interval, List<Intervalable> overlaps, List<Intervalable> newOverlaps) {
+        for (Intervalable currentInterval : newOverlaps) {
+            if (!currentInterval.equals(interval)) {
+                overlaps.add(currentInterval);
+            }
+        }
+    }
+
+    protected List<Intervalable> checkForOverlapsToTheLeft(Intervalable interval) {
+        return checkForOverlaps(interval, Direction.LEFT);
+    }
+
+    protected List<Intervalable> checkForOverlapsToTheRight(Intervalable interval) {
+        return checkForOverlaps(interval, Direction.RIGHT);
+    }
+
+    protected List<Intervalable> checkForOverlaps(Intervalable interval, Direction direction) {
+
+        List<Intervalable> overlaps = new ArrayList<Intervalable>();
+        for (Intervalable currentInterval : this.intervals) {
+            switch (direction) {
+                case LEFT :
+                    if (currentInterval.getStart() <= interval.getEnd()) {
+                        overlaps.add(currentInterval);
+                    }
+                    break;
+                case RIGHT :
+                    if (currentInterval.getEnd() >= interval.getStart()) {
+                        overlaps.add(currentInterval);
+                    }
+                    break;
+            }
+        }
+        return overlaps;
+    }
+
+
+    protected List<Intervalable> findOverlappingRanges(IntervalNode node, Intervalable interval) {
+        if (node != null) {
+            return node.findOverlaps(interval);
+        }
+        return Collections.emptyList();
+    }
+
+}

src/main/java/org/ahocorasick/interval/IntervalTree.java

@@ -0,0 +1,48 @@
+package org.ahocorasick.interval;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+import java.util.TreeSet;
+
+public class IntervalTree {
+
+    private IntervalNode rootNode = null;
+
+    public IntervalTree(List<Intervalable> intervals) {
+        this.rootNode = new IntervalNode(intervals);
+    }
+
+    public List<Intervalable> removeOverlaps(List<Intervalable> intervals) {
+
+        // Sort the intervals on size, then left-most position
+        Collections.sort(intervals, new IntervalableComparatorBySize());
+
+        Set<Intervalable> removeIntervals = new TreeSet<Intervalable>();
+
+        for (Intervalable interval : intervals) {
+            // If the interval was already removed, ignore it
+            if (removeIntervals.contains(interval)) {
+                continue;
+            }
+
+            // Remove all overallping intervals
+            removeIntervals.addAll(findOverlaps(interval));
+        }
+
+        // Remove all intervals that were overlapping
+        for (Intervalable removeInterval : removeIntervals) {
+            intervals.remove(removeInterval);
+        }
+
+        // Sort the intervals, now on left-most position only
+        Collections.sort(intervals, new IntervalableComparatorByPosition());
+
+        return intervals;
+    }
+
+    public List<Intervalable> findOverlaps(Intervalable interval) {
+        return rootNode.findOverlaps(interval);
+    }
+
+}

src/main/java/org/ahocorasick/interval/Intervalable.java

@@ -0,0 +1,9 @@
+package org.ahocorasick.interval;
+
+public interface Intervalable extends Comparable {
+
+    public int getStart();
+    public int getEnd();
+    public int size();
+
+}

src/main/java/org/ahocorasick/interval/IntervalableComparatorByPosition.java

@@ -0,0 +1,12 @@
+package org.ahocorasick.interval;
+
+import java.util.Comparator;
+
+public class IntervalableComparatorByPosition implements Comparator<Intervalable> {
+
+    @Override
+    public int compare(Intervalable intervalable, Intervalable intervalable2) {
+        return intervalable.getStart() - intervalable2.getStart();
+    }
+
+}

src/main/java/org/ahocorasick/interval/IntervalableComparatorBySize.java

@@ -0,0 +1,16 @@
+package org.ahocorasick.interval;
+
+import java.util.Comparator;
+
+public class IntervalableComparatorBySize implements Comparator<Intervalable> {
+
+    @Override
+    public int compare(Intervalable intervalable, Intervalable intervalable2) {
+        int comparison = intervalable2.size() - intervalable.size();
+        if (comparison == 0) {
+            comparison = intervalable.getStart() - intervalable2.getStart();
+        }
+        return comparison;
+    }
+
+}

src/main/java/org/ahocorasick/trie/Emit.java

@@ -0,0 +1,19 @@
+package org.ahocorasick.trie;
+
+import org.ahocorasick.interval.Interval;
+import org.ahocorasick.interval.Intervalable;
+
+public class Emit extends Interval implements Intervalable {
+
+    private final String keyword;
+
+    public Emit(final int start, final int end, final String keyword) {
+        super(start, end);
+        this.keyword = keyword;
+    }
+
+    public String getKeyword() {
+        return this.keyword;
+    }
+
+}

src/main/java/org/ahocorasick/trie/State.java

@@ -1,4 +1,4 @@
-package org.ahocorasick;
+package org.ahocorasick.trie;
 
 import java.util.*;
 

src/main/java/org/ahocorasick/trie/Trie.java

@@ -1,4 +1,7 @@
-package org.ahocorasick;
+package org.ahocorasick.trie;
+
+import org.ahocorasick.interval.IntervalTree;
+import org.ahocorasick.interval.Intervalable;
 
 import java.util.ArrayList;
 import java.util.Collection;
@@ -13,14 +16,26 @@ import java.util.concurrent.LinkedBlockingDeque;
  */
 public class Trie {
 
+    private TrieConfig trieConfig;
+
     private State rootState;
 
     private boolean failureStatesConstructed = false;
 
-    public Trie() {
+    public Trie(TrieConfig trieConfig) {
+        this.trieConfig = trieConfig;
         this.rootState = new State();
     }
 
+    public Trie() {
+        this(new TrieConfig());
+    }
+
+    public Trie removeOverlaps() {
+        this.trieConfig.setAllowOverlaps(false);
+        return this;
+    }
+
     public void addKeyword(String keyword) {
 
         State currentState = this.rootState;
@@ -30,6 +45,7 @@ public class Trie {
         currentState.addEmit(keyword);
     }
 
+    @SuppressWarnings("unchecked")
     public Collection<Emit> parseText(String text) {
         checkForConstructedFailureStates();
 
@@ -41,6 +57,12 @@ public class Trie {
             storeEmits(position, currentState, collectedEmits);
             position++;
         }
+
+        if (!trieConfig.isAllowOverlaps()) {
+            IntervalTree intervalTree = new IntervalTree((List<Intervalable>)(List<?>)collectedEmits);
+            intervalTree.removeOverlaps((List<Intervalable>) (List<?>) collectedEmits);
+        }
+
         return collectedEmits;
     }
 

src/main/java/org/ahocorasick/trie/TrieConfig.java

@@ -0,0 +1,15 @@
+package org.ahocorasick.trie;
+
+public class TrieConfig {
+
+    private boolean allowOverlaps = true;
+
+    public boolean isAllowOverlaps() {
+        return allowOverlaps;
+    }
+
+    public void setAllowOverlaps(boolean allowOverlaps) {
+        this.allowOverlaps = allowOverlaps;
+    }
+
+}

src/test/java/org/ahocorasick/interval/IntervalTest.java

@@ -0,0 +1,57 @@
+package org.ahocorasick.interval;
+
+import org.junit.Test;
+
+import java.util.*;
+
+import static junit.framework.Assert.assertEquals;
+import static junit.framework.Assert.assertFalse;
+import static junit.framework.Assert.assertTrue;
+
+public class IntervalTest {
+
+    @Test
+    public void construct() {
+        Interval i = new Interval(1,3);
+        assertEquals(1, i.getStart());
+        assertEquals(3, i.getEnd());
+    }
+
+    @Test
+    public void size() {
+        assertEquals(3, new Interval(0,2).size());
+    }
+
+    @Test
+    public void intervaloverlaps() {
+        assertTrue(new Interval(1,3).overlapsWith(new Interval(2,4)));
+    }
+
+    @Test
+    public void intervalDoesNotOverlap() {
+        assertFalse(new Interval(1, 13).overlapsWith(new Interval(27, 42)));
+    }
+
+    @Test
+    public void pointOverlaps() {
+        assertTrue(new Interval(1,3).overlapsWith(2));
+    }
+
+    @Test
+    public void pointDoesNotOverlap() {
+        assertFalse(new Interval(1, 13).overlapsWith(42));
+    }
+
+    @Test
+    public void comparable() {
+        Set<Interval> intervals = new TreeSet<Interval>();
+        intervals.add(new Interval(4, 6));
+        intervals.add(new Interval(2, 7));
+        intervals.add(new Interval(3, 4));
+        Iterator<Interval> it = intervals.iterator();
+        assertEquals(2, it.next().getStart());
+        assertEquals(3, it.next().getStart());
+        assertEquals(4, it.next().getStart());
+    }
+
+}

src/test/java/org/ahocorasick/interval/IntervalTreeTest.java

@@ -0,0 +1,51 @@
+package org.ahocorasick.interval;
+
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import static junit.framework.Assert.assertEquals;
+
+public class IntervalTreeTest {
+    
+    @Test
+    public void findOverlaps() {
+        List<Intervalable> intervals = new ArrayList<Intervalable>();
+        intervals.add(new Interval(0, 2));
+        intervals.add(new Interval(1, 3));
+        intervals.add(new Interval(2, 4));
+        intervals.add(new Interval(3, 5));
+        intervals.add(new Interval(4, 6));
+        intervals.add(new Interval(5, 7));
+        IntervalTree intervalTree = new IntervalTree(intervals);
+        List<Intervalable> overlaps = intervalTree.findOverlaps(new Interval(1,3));
+        assertEquals(3, overlaps.size());
+        Iterator<Intervalable> overlapsIt = overlaps.iterator();
+        assertOverlap(overlapsIt.next(), 2, 4);
+        assertOverlap(overlapsIt.next(), 3, 5);
+        assertOverlap(overlapsIt.next(), 0, 2);
+    }
+
+    @Test
+    public void removeOverlaps() {
+        List<Intervalable> intervals = new ArrayList<Intervalable>();
+        intervals.add(new Interval(0, 2));
+        intervals.add(new Interval(4, 5));
+        intervals.add(new Interval(2, 10));
+        intervals.add(new Interval(6, 13));
+        intervals.add(new Interval(9, 15));
+        intervals.add(new Interval(12, 16));
+        IntervalTree intervalTree = new IntervalTree(intervals);
+        intervals = intervalTree.removeOverlaps(intervals);
+        assertEquals(2, intervals.size());
+
+    }
+
+    protected void assertOverlap(Intervalable interval, int expectedStart, int expectedEnd) {
+        assertEquals(expectedStart, interval.getStart());
+        assertEquals(expectedEnd, interval.getEnd());
+    }
+    
+}

src/test/java/org/ahocorasick/interval/IntervalableComparatorByPositionTest.java

@@ -0,0 +1,25 @@
+package org.ahocorasick.interval;
+
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import static junit.framework.Assert.assertEquals;
+
+public class IntervalableComparatorByPositionTest {
+
+    @Test
+    public void sortOnPosition() {
+        List<Intervalable> intervals = new ArrayList<Intervalable>();
+        intervals.add(new Interval(4,5));
+        intervals.add(new Interval(1,4));
+        intervals.add(new Interval(3,8));
+        Collections.sort(intervals, new IntervalableComparatorByPosition());
+        assertEquals(4, intervals.get(0).size());
+        assertEquals(6, intervals.get(1).size());
+        assertEquals(2, intervals.get(2).size());
+    }
+
+}

src/test/java/org/ahocorasick/interval/IntervalableComparatorBySizeTest.java

@@ -0,0 +1,35 @@
+package org.ahocorasick.interval;
+
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import static junit.framework.Assert.assertEquals;
+
+public class IntervalableComparatorBySizeTest {
+
+    @Test
+    public void sortOnSize() {
+        List<Intervalable> intervals = new ArrayList<Intervalable>();
+        intervals.add(new Interval(4,5));
+        intervals.add(new Interval(1,4));
+        intervals.add(new Interval(3,8));
+        Collections.sort(intervals, new IntervalableComparatorBySize());
+        assertEquals(6, intervals.get(0).size());
+        assertEquals(4, intervals.get(1).size());
+        assertEquals(2, intervals.get(2).size());
+    }
+
+    @Test
+    public void sortOnSizeThenPosition() {
+        List<Intervalable> intervals = new ArrayList<Intervalable>();
+        intervals.add(new Interval(4,7));
+        intervals.add(new Interval(2,5));
+        Collections.sort(intervals, new IntervalableComparatorBySize());
+        assertEquals(2, intervals.get(0).getStart());
+        assertEquals(4, intervals.get(1).getStart());
+    }
+
+}

src/test/java/org/ahocorasick/trie/EmitTest.java

@@ -0,0 +1,24 @@
+package org.ahocorasick.trie;
+
+import org.junit.Test;
+
+import static junit.framework.Assert.assertEquals;
+import static junit.framework.Assert.assertNotSame;
+
+public class EmitTest {
+
+    @Test
+    public void equals() {
+        Emit one = new Emit(13, 42, null);
+        Emit two = new Emit(13, 42, null);
+        assertEquals(one, two);
+    }
+
+    @Test
+    public void notEquals() {
+        Emit one = new Emit(13, 42, null);
+        Emit two = new Emit(13, 43, null);
+        assertNotSame(one, two);
+    }
+
+}

src/test/java/org/ahocorasick/trie/StateTest.java

@@ -1,5 +1,6 @@
-package org.ahocorasick;
+package org.ahocorasick.trie;
 
+import org.ahocorasick.trie.State;
 import org.junit.Test;
 
 import static junit.framework.Assert.assertEquals;

src/test/java/org/ahocorasick/trie/TrieTest.java

@@ -1,5 +1,7 @@
-package org.ahocorasick;
+package org.ahocorasick.trie;
 
+import org.ahocorasick.trie.Emit;
+import org.ahocorasick.trie.Trie;
 import org.junit.Test;
 
 import java.util.Collection;
@@ -77,6 +79,35 @@ public class TrieTest {
         checkEmit(iterator.next(), 51, 58, "broccoli");
     }
 
+    @Test
+    public void longAndShortOverlappingMatch() {
+        Trie trie = new Trie();
+        trie.addKeyword("he");
+        trie.addKeyword("hehehehe");
+        Collection<Emit> emits = trie.parseText("hehehehehe");
+        Iterator<Emit> iterator = emits.iterator();
+        checkEmit(iterator.next(), 0, 1, "he");
+        checkEmit(iterator.next(), 2, 3, "he");
+        checkEmit(iterator.next(), 4, 5, "he");
+        checkEmit(iterator.next(), 0, 7, "hehehehe");
+        checkEmit(iterator.next(), 6, 7, "he");
+        checkEmit(iterator.next(), 2, 9, "hehehehe");
+        checkEmit(iterator.next(), 8, 9, "he");
+    }
+
+    @Test
+    public void nonOverlapping() {
+        Trie trie = new Trie().removeOverlaps();
+        trie.addKeyword("ab");
+        trie.addKeyword("cba");
+        trie.addKeyword("ababc");
+        Collection<Emit> emits = trie.parseText("ababcbab");
+        Iterator<Emit> iterator = emits.iterator();
+        // With overlaps: ab@1, ab@3, ababc@4, cba@6, ab@7
+        checkEmit(iterator.next(), 0, 4, "ababc");
+        checkEmit(iterator.next(), 6, 7, "ab");
+    }
+
     private void checkEmit(Emit next, int expectedStart, int expectedEnd, String expectedKeyword) {
         assertEquals(expectedStart, next.getStart());
         assertEquals(expectedEnd, next.getEnd());