RED-2845 Bugfix: Implemented EntitySearchUtils.findNonOverlappingMatchEntities()

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java

@@ -216,8 +216,9 @@ public class Section {
             while (matcher.find()) {
                 String match = matcher.group(group);
 
-                if (!overlapsMatchEntities(entities, match)) {
+                if (StringUtils.isNotBlank(match)) {
-                    expanded.addAll(findEntities(entity.getWord() + match, type, false, entity.isRedaction(), entity.getMatchedRule(), entity.getRedactionReason(), entity.getLegalBasis()));
+                    Set<Entity> expandedEntities = findEntities(entity.getWord() + match, type, false, entity.isRedaction(), entity.getMatchedRule(), entity.getRedactionReason(), entity.getLegalBasis());
+                    expanded.addAll(EntitySearchUtils.findNonOverlappingMatchEntities(entities, expandedEntities));
                 }
             }
         }
@@ -227,19 +228,6 @@ public class Section {
     }
 
 
-    private boolean overlapsMatchEntities(Set<Entity> entities, String match) {
-
-        if (entities != null && StringUtils.isNotBlank(match)) {
-            for (Entity entity : entities) {
-                if (StringUtils.containsIgnoreCase(entity.getWord(), match.trim()) && entity.getEnd() + 1 >= entity.getStart()) {
-                    return true;
-                }
-            }
-        }
-        return false;
-    }
-
-
     @ThenAction
     public void redactImage(@Argument(ArgumentType.TYPE) String type,
                             @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java

@@ -1,5 +1,17 @@
 package com.iqser.red.service.redaction.v1.server.redaction.utils;
 
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
 import com.iqser.red.service.redaction.v1.model.Engine;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
 import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrementValue;
@@ -10,10 +22,6 @@ import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
 import lombok.experimental.UtilityClass;
 import lombok.extern.slf4j.Slf4j;
 
-import java.util.*;
-import java.util.regex.Pattern;
-import java.util.stream.Collectors;
-
 @Slf4j
 @UtilityClass
 @SuppressWarnings("PMD")
@@ -37,8 +45,7 @@ public class EntitySearchUtils {
                 startIndex = inputString.indexOf(cleanValue, stopIndex);
                 stopIndex = startIndex + cleanValue.length();
 
-                if (startIndex > -1 && (startIndex == 0 || Character.isWhitespace(inputString.charAt(startIndex - 1)) || isSeparator(inputString
+                if (startIndex > -1 && (startIndex == 0 || Character.isWhitespace(inputString.charAt(startIndex - 1)) || isSeparator(inputString.charAt(startIndex - 1))) && (stopIndex == inputString.length() || isSeparator(inputString.charAt(stopIndex)))) {
-                        .charAt(startIndex - 1))) && (stopIndex == inputString.length() || isSeparator(inputString.charAt(stopIndex)))) {
                     return true;
                 }
             } while (startIndex > -1);
@@ -66,8 +73,7 @@ public class EntitySearchUtils {
                 startIndex = inputString.indexOf(cleanValue, stopIndex);
                 stopIndex = startIndex + cleanValue.length();
 
-                if (startIndex > -1 && (startIndex == 0 || Character.isWhitespace(inputString.charAt(startIndex - 1)) || isSeparator(inputString
+                if (startIndex > -1 && (startIndex == 0 || Character.isWhitespace(inputString.charAt(startIndex - 1)) || isSeparator(inputString.charAt(startIndex - 1))) && (stopIndex == inputString.length() || isSeparator(inputString.charAt(stopIndex)))) {
-                        .charAt(startIndex - 1))) && (stopIndex == inputString.length() || isSeparator(inputString.charAt(stopIndex)))) {
                     found.add(new Entity(inputString.substring(startIndex, stopIndex), type, startIndex, stopIndex, headline, sectionNumber, isDictionaryEntry, isDossierDictionary, engine));
                 }
             } while (startIndex > -1);
@@ -121,8 +127,7 @@ public class EntitySearchUtils {
         for (Entity word : entities) {
             for (Entity inner : entities) {
                 if (inner.getWord().length() < word.getWord()
-                        .length() && inner.getStart() >= word.getStart() && inner.getEnd() <= word.getEnd() && word != inner && word
+                        .length() && inner.getStart() >= word.getStart() && inner.getEnd() <= word.getEnd() && word != inner && word.getSectionNumber() == inner.getSectionNumber()) {
-                        .getSectionNumber() == inner.getSectionNumber()) {
                     wordsToRemove.add(inner);
                 }
             }
@@ -169,7 +174,9 @@ public class EntitySearchUtils {
 
         for (Entity toAdd : toBeAdded) {
             if (existing.contains(toAdd)) {
-                Optional<Entity> existingOptional  = existing.stream().filter(entity -> entity.equals(toAdd)).findFirst();
+                Optional<Entity> existingOptional = existing.stream()
+                        .filter(entity -> entity.equals(toAdd))
+                        .findFirst();
                 if (!existingOptional.isPresent()) {
                     return;
                 }
@@ -181,4 +188,20 @@ public class EntitySearchUtils {
         }
     }
 
+
+    public Set<Entity> findNonOverlappingMatchEntities(Set<Entity> existingEntities, Set<Entity> foundEntities) {
+
+        Set<Entity> result = new HashSet<>();
+        if (existingEntities != null && foundEntities != null) {
+            for (Entity existingEntity : existingEntities) {
+                for (Entity foundEntity : foundEntities) {
+                    if (existingEntity.getEnd() < foundEntity.getStart() || foundEntity.getEnd() < existingEntity.getStart()) {
+                        result.add(foundEntity);
+                    }
+                }
+            }
+        }
+        return result;
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtilsTest.java

@@ -27,4 +27,164 @@ public class EntitySearchUtilsTest {
 
     }
 
+
+    /*
+     * Text:     Batman X. Superman Y.
+     * Position: 0123456789
+     *                     0123456789
+     *                               0123456789
+     */
+    @Test
+    public void testNotOverlappingEntitiesExpandedEnd() {
+
+        // Arrange
+        Set<Entity> existingEntities = new HashSet<>();
+        Entity existingEntity1 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, false, false, Engine.RULE);
+        Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE);
+        existingEntities.add(existingEntity1);
+        existingEntities.add(existingEntity2);
+
+        Set<Entity> foundEntities = new HashSet<>();
+        Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE);
+        Entity foundEntities2 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, false, false, Engine.RULE);
+        foundEntities.add(foundEntities1);
+        foundEntities.add(foundEntities2);
+
+        // Act
+        Set<Entity> result = EntitySearchUtils.findNonOverlappingMatchEntities(existingEntities, foundEntities);
+
+        // Assert
+        assertThat(result.size()).isEqualTo(2);
+        assertThat(result).contains(foundEntities1);
+        assertThat(result).contains(foundEntities2);
+
+    }
+
+
+    /*
+     * Text:     Batman X. Superman Y.
+     * Position: 0123456789
+     *                     0123456789
+     *                               0123456789
+     */
+    @Test
+    public void testNotOverlappingEntitiesExpandedStartAndEndOverlapping() {
+
+        // Arrange
+        Set<Entity> existingEntities = new HashSet<>();
+        Entity existingEntity1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE);
+        Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE);
+        existingEntities.add(existingEntity1);
+        existingEntities.add(existingEntity2);
+
+        Set<Entity> foundEntities = new HashSet<>();
+        Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE);
+        Entity foundEntities2 = new Entity("X. Superman Y.", "fake type", 7, 20, "fake headline", 0, false, false, Engine.RULE);
+        foundEntities.add(foundEntities1);
+        foundEntities.add(foundEntities2);
+
+        // Act
+        Set<Entity> result = EntitySearchUtils.findNonOverlappingMatchEntities(existingEntities, foundEntities);
+
+        // Assert
+        assertThat(result.size()).isEqualTo(1);
+        assertThat(result).contains(foundEntities1);
+
+    }
+
+
+    /*
+     * Text:     Batman X. Superman Y.
+     * Position: 0123456789
+     *                     0123456789
+     *                               0123456789
+     */
+    @Test
+    public void testNotOverlappingEntitiesExpandedStartAndEnd() {
+
+        // Arrange
+        Set<Entity> existingEntities = new HashSet<>();
+        Entity existingEntity1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE);
+        Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE);
+        existingEntities.add(existingEntity1);
+        existingEntities.add(existingEntity2);
+
+        Set<Entity> foundEntities = new HashSet<>();
+        Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE);
+        Entity foundEntities2 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, false, false, Engine.RULE);
+        foundEntities.add(foundEntities1);
+        foundEntities.add(foundEntities2);
+
+        // Act
+        Set<Entity> result = EntitySearchUtils.findNonOverlappingMatchEntities(existingEntities, foundEntities);
+
+        // Assert
+        assertThat(result.size()).isEqualTo(1);
+        assertThat(result).contains(foundEntities1);
+
+    }
+
+
+    /*
+     * Text:     Batman X. Superman Y.
+     * Position: 0123456789
+     *                     0123456789
+     *                               0123456789
+     */
+    @Test
+    public void testNotOverlappingEntitiesExpandedExistingAndExpandedEnd() {
+
+        // Arrange
+        Set<Entity> existingEntities = new HashSet<>();
+        Entity existingEntity1 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, false, false, Engine.RULE);
+        Entity existingEntity2 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, false, false, Engine.RULE);
+        existingEntities.add(existingEntity1);
+        existingEntities.add(existingEntity2);
+
+        Set<Entity> foundEntities = new HashSet<>();
+        Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE);
+        Entity foundEntities2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE);
+        foundEntities.add(foundEntities1);
+        foundEntities.add(foundEntities2);
+
+        // Act
+        Set<Entity> result = EntitySearchUtils.findNonOverlappingMatchEntities(existingEntities, foundEntities);
+
+        // Assert
+        assertThat(result.size()).isEqualTo(1);
+        assertThat(result).contains(foundEntities2);
+
+    }
+
+    /*
+     * Text:     Batman X. Superman Y.
+     * Position: 0123456789
+     *                     0123456789
+     *                               0123456789
+     */
+    @Test
+    public void testNotOverlappingEntitiesExpandedEndLong() {
+
+        // Arrange
+        Set<Entity> existingEntities = new HashSet<>();
+        Entity existingEntity1 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, false, false, Engine.RULE);
+        Entity existingEntity2 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, false, false, Engine.RULE);
+        existingEntities.add(existingEntity1);
+        existingEntities.add(existingEntity2);
+
+        Set<Entity> foundEntities = new HashSet<>();
+        Entity foundEntities1 = new Entity("Batman X. Superman", "fake type", 0, 17, "fake headline", 0, false, false, Engine.RULE);
+        Entity foundEntities2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE);
+        foundEntities.add(foundEntities1);
+        foundEntities.add(foundEntities2);
+
+        // Act
+        Set<Entity> result = EntitySearchUtils.findNonOverlappingMatchEntities(existingEntities, foundEntities);
+
+        // Assert
+        assertThat(result.size()).isEqualTo(1);
+        assertThat(result).contains(foundEntities2);
+
+    }
+
 }