diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java index f1540ace..0592b7a3 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java @@ -216,8 +216,9 @@ public class Section { while (matcher.find()) { String match = matcher.group(group); - if (!overlapsMatchEntities(entities, match)) { - expanded.addAll(findEntities(entity.getWord() + match, type, false, entity.isRedaction(), entity.getMatchedRule(), entity.getRedactionReason(), entity.getLegalBasis())); + if (StringUtils.isNotBlank(match)) { + Set expandedEntities = findEntities(entity.getWord() + match, type, false, entity.isRedaction(), entity.getMatchedRule(), entity.getRedactionReason(), entity.getLegalBasis()); + expanded.addAll(EntitySearchUtils.findNonOverlappingMatchEntities(entities, expandedEntities)); } } } @@ -227,19 +228,6 @@ public class Section { } - private boolean overlapsMatchEntities(Set entities, String match) { - - if (entities != null && StringUtils.isNotBlank(match)) { - for (Entity entity : entities) { - if (StringUtils.containsIgnoreCase(entity.getWord(), match.trim()) && entity.getEnd() + 1 >= entity.getStart()) { - return true; - } - } - } - return false; - } - - @ThenAction public void redactImage(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java index d514df1e..7d101356 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java @@ -1,5 +1,17 @@ package com.iqser.red.service.redaction.v1.server.redaction.utils; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + import com.iqser.red.service.redaction.v1.model.Engine; import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary; import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrementValue; @@ -10,10 +22,6 @@ import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText; import lombok.experimental.UtilityClass; import lombok.extern.slf4j.Slf4j; -import java.util.*; -import java.util.regex.Pattern; -import java.util.stream.Collectors; - @Slf4j @UtilityClass @SuppressWarnings("PMD") @@ -37,8 +45,7 @@ public class EntitySearchUtils { startIndex = inputString.indexOf(cleanValue, stopIndex); stopIndex = startIndex + cleanValue.length(); - if (startIndex > -1 && (startIndex == 0 || Character.isWhitespace(inputString.charAt(startIndex - 1)) || isSeparator(inputString - .charAt(startIndex - 1))) && (stopIndex == inputString.length() || isSeparator(inputString.charAt(stopIndex)))) { + if (startIndex > -1 && (startIndex == 0 || Character.isWhitespace(inputString.charAt(startIndex - 1)) || isSeparator(inputString.charAt(startIndex - 1))) && (stopIndex == inputString.length() || isSeparator(inputString.charAt(stopIndex)))) { return true; } } while (startIndex > -1); @@ -66,8 +73,7 @@ public class EntitySearchUtils { startIndex = inputString.indexOf(cleanValue, stopIndex); stopIndex = startIndex + cleanValue.length(); - if (startIndex > -1 && (startIndex == 0 || Character.isWhitespace(inputString.charAt(startIndex - 1)) || isSeparator(inputString - .charAt(startIndex - 1))) && (stopIndex == inputString.length() || isSeparator(inputString.charAt(stopIndex)))) { + if (startIndex > -1 && (startIndex == 0 || Character.isWhitespace(inputString.charAt(startIndex - 1)) || isSeparator(inputString.charAt(startIndex - 1))) && (stopIndex == inputString.length() || isSeparator(inputString.charAt(stopIndex)))) { found.add(new Entity(inputString.substring(startIndex, stopIndex), type, startIndex, stopIndex, headline, sectionNumber, isDictionaryEntry, isDossierDictionary, engine)); } } while (startIndex > -1); @@ -121,8 +127,7 @@ public class EntitySearchUtils { for (Entity word : entities) { for (Entity inner : entities) { if (inner.getWord().length() < word.getWord() - .length() && inner.getStart() >= word.getStart() && inner.getEnd() <= word.getEnd() && word != inner && word - .getSectionNumber() == inner.getSectionNumber()) { + .length() && inner.getStart() >= word.getStart() && inner.getEnd() <= word.getEnd() && word != inner && word.getSectionNumber() == inner.getSectionNumber()) { wordsToRemove.add(inner); } } @@ -141,14 +146,14 @@ public class EntitySearchUtils { if (entities.contains(found)) { Optional existingOptional = entities.stream().filter(entity -> entity.equals(found)).findFirst(); - if(!existingOptional.isPresent()){ + if (!existingOptional.isPresent()) { return; } var existing = existingOptional.get(); - if(existing.getType().equals(found.getType())){ + if (existing.getType().equals(found.getType())) { existing.getEngines().addAll(found.getEngines()); - } else if (dictionary.getDictionaryRank(existing.getType()) <= dictionary.getDictionaryRank(found.getType())){ + } else if (dictionary.getDictionaryRank(existing.getType()) <= dictionary.getDictionaryRank(found.getType())) { entities.remove(found); entities.add(found); } @@ -165,12 +170,14 @@ public class EntitySearchUtils { } - public void addOrAddEngine(Set existing, Set toBeAdded){ + public void addOrAddEngine(Set existing, Set toBeAdded) { - for(Entity toAdd: toBeAdded){ + for (Entity toAdd : toBeAdded) { if (existing.contains(toAdd)) { - Optional existingOptional = existing.stream().filter(entity -> entity.equals(toAdd)).findFirst(); - if(!existingOptional.isPresent()){ + Optional existingOptional = existing.stream() + .filter(entity -> entity.equals(toAdd)) + .findFirst(); + if (!existingOptional.isPresent()) { return; } var existingEntity = existingOptional.get(); @@ -181,4 +188,20 @@ public class EntitySearchUtils { } } + + public Set findNonOverlappingMatchEntities(Set existingEntities, Set foundEntities) { + + Set result = new HashSet<>(); + if (existingEntities != null && foundEntities != null) { + for (Entity existingEntity : existingEntities) { + for (Entity foundEntity : foundEntities) { + if (existingEntity.getEnd() < foundEntity.getStart() || foundEntity.getEnd() < existingEntity.getStart()) { + result.add(foundEntity); + } + } + } + } + return result; + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtilsTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtilsTest.java index ae2fb019..eec7d419 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtilsTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtilsTest.java @@ -27,4 +27,164 @@ public class EntitySearchUtilsTest { } + + /* + * Text: Batman X. Superman Y. + * Position: 0123456789 + * 0123456789 + * 0123456789 + */ + @Test + public void testNotOverlappingEntitiesExpandedEnd() { + + // Arrange + Set existingEntities = new HashSet<>(); + Entity existingEntity1 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, false, false, Engine.RULE); + Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE); + existingEntities.add(existingEntity1); + existingEntities.add(existingEntity2); + + Set foundEntities = new HashSet<>(); + Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE); + Entity foundEntities2 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, false, false, Engine.RULE); + foundEntities.add(foundEntities1); + foundEntities.add(foundEntities2); + + // Act + Set result = EntitySearchUtils.findNonOverlappingMatchEntities(existingEntities, foundEntities); + + // Assert + assertThat(result.size()).isEqualTo(2); + assertThat(result).contains(foundEntities1); + assertThat(result).contains(foundEntities2); + + } + + + /* + * Text: Batman X. Superman Y. + * Position: 0123456789 + * 0123456789 + * 0123456789 + */ + @Test + public void testNotOverlappingEntitiesExpandedStartAndEndOverlapping() { + + // Arrange + Set existingEntities = new HashSet<>(); + Entity existingEntity1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE); + Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE); + existingEntities.add(existingEntity1); + existingEntities.add(existingEntity2); + + Set foundEntities = new HashSet<>(); + Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE); + Entity foundEntities2 = new Entity("X. Superman Y.", "fake type", 7, 20, "fake headline", 0, false, false, Engine.RULE); + foundEntities.add(foundEntities1); + foundEntities.add(foundEntities2); + + // Act + Set result = EntitySearchUtils.findNonOverlappingMatchEntities(existingEntities, foundEntities); + + // Assert + assertThat(result.size()).isEqualTo(1); + assertThat(result).contains(foundEntities1); + + } + + + /* + * Text: Batman X. Superman Y. + * Position: 0123456789 + * 0123456789 + * 0123456789 + */ + @Test + public void testNotOverlappingEntitiesExpandedStartAndEnd() { + + // Arrange + Set existingEntities = new HashSet<>(); + Entity existingEntity1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE); + Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE); + existingEntities.add(existingEntity1); + existingEntities.add(existingEntity2); + + Set foundEntities = new HashSet<>(); + Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE); + Entity foundEntities2 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, false, false, Engine.RULE); + foundEntities.add(foundEntities1); + foundEntities.add(foundEntities2); + + // Act + Set result = EntitySearchUtils.findNonOverlappingMatchEntities(existingEntities, foundEntities); + + // Assert + assertThat(result.size()).isEqualTo(1); + assertThat(result).contains(foundEntities1); + + } + + + /* + * Text: Batman X. Superman Y. + * Position: 0123456789 + * 0123456789 + * 0123456789 + */ + @Test + public void testNotOverlappingEntitiesExpandedExistingAndExpandedEnd() { + + // Arrange + Set existingEntities = new HashSet<>(); + Entity existingEntity1 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, false, false, Engine.RULE); + Entity existingEntity2 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, false, false, Engine.RULE); + existingEntities.add(existingEntity1); + existingEntities.add(existingEntity2); + + Set foundEntities = new HashSet<>(); + Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE); + Entity foundEntities2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE); + foundEntities.add(foundEntities1); + foundEntities.add(foundEntities2); + + // Act + Set result = EntitySearchUtils.findNonOverlappingMatchEntities(existingEntities, foundEntities); + + // Assert + assertThat(result.size()).isEqualTo(1); + assertThat(result).contains(foundEntities2); + + } + + /* + * Text: Batman X. Superman Y. + * Position: 0123456789 + * 0123456789 + * 0123456789 + */ + @Test + public void testNotOverlappingEntitiesExpandedEndLong() { + + // Arrange + Set existingEntities = new HashSet<>(); + Entity existingEntity1 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, false, false, Engine.RULE); + Entity existingEntity2 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, false, false, Engine.RULE); + existingEntities.add(existingEntity1); + existingEntities.add(existingEntity2); + + Set foundEntities = new HashSet<>(); + Entity foundEntities1 = new Entity("Batman X. Superman", "fake type", 0, 17, "fake headline", 0, false, false, Engine.RULE); + Entity foundEntities2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE); + foundEntities.add(foundEntities1); + foundEntities.add(foundEntities2); + + // Act + Set result = EntitySearchUtils.findNonOverlappingMatchEntities(existingEntities, foundEntities); + + // Assert + assertThat(result.size()).isEqualTo(1); + assertThat(result).contains(foundEntities2); + + } + }