diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SearchableText.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SearchableText.java index 60565609..05561f3e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SearchableText.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SearchableText.java @@ -10,6 +10,7 @@ import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizati import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.regex.Pattern; public class SearchableText { @@ -94,8 +95,8 @@ public class SearchableText { .get(i) .charAt(j, caseInsensitive) == '-') { - if (counter != 0 || i == 0 && j == 0 || j != 0 && SeparatorUtils.isSeparator(searchSpace.get(i) - .charAt(j - 1, caseInsensitive)) || j == 0 && i != 0 && SeparatorUtils.isSeparator(searchSpace.get(i - 1) + if (counter != 0 || i == 0 && j == 0 || j != 0 && isSeparator(searchSpace.get(i) + .charAt(j - 1, caseInsensitive)) || j == 0 && i != 0 && isSeparator(searchSpace.get(i - 1) .charAt(searchSpace.get(i - 1) .length() - 1, caseInsensitive)) || j == 0 && i != 0 && searchSpace.get(i - 1) .charAt(searchSpace.get(i - 1).length() - 1, caseInsensitive) != ' ' && searchSpace.get(i) @@ -111,9 +112,9 @@ public class SearchableText { crossSequenceParts.add(partMatch); if (i == searchSpace.size() - 1 && j == searchSpace.get(i) - .length() - 1 || j != searchSpace.get(i).length() - 1 && SeparatorUtils.isSeparator(searchSpace.get(i) + .length() - 1 || j != searchSpace.get(i).length() - 1 && isSeparator(searchSpace.get(i) .charAt(j + 1, caseInsensitive)) || j == searchSpace.get(i) - .length() - 1 && SeparatorUtils.isSeparator(searchSpace.get(i + 1) + .length() - 1 && isSeparator(searchSpace.get(i + 1) .charAt(0, caseInsensitive)) || j == searchSpace.get(i) .length() - 1 && searchSpace.get(i) .charAt(j, caseInsensitive) != ' ' && searchSpace.get(i + 1) @@ -180,6 +181,11 @@ public class SearchableText { return result; } + private boolean isSeparator(char c) { + + return Character.isWhitespace(c) || Pattern.matches("\\p{Punct}", String.valueOf(c)) || c == '\"' || c == '‘' || c == '’' || c == '”'; + } + @Override public String toString() { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java index 267bd156..0e4470f9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java @@ -46,14 +46,18 @@ public class EntitySearchUtils { private void validateAndAddEntity(Set entities, FindEntityDetails findEntityDetails, String inputString, int startIndex, int stopIndex) { - if ((startIndex == 0 || SeparatorUtils.isSeparator(inputString.charAt(startIndex - 1))) - && (stopIndex == inputString.length() || SeparatorUtils.isSeparator(inputString.charAt(stopIndex)))) { + if (startIndex > -1 && (startIndex == 0 || Character.isWhitespace(inputString.charAt(startIndex - 1)) || isSeparator(inputString.charAt(startIndex - 1))) && (stopIndex == inputString.length() || isSeparator(inputString.charAt(stopIndex)))) { entities.add(new Entity(inputString.substring(startIndex, stopIndex), findEntityDetails.getType(), startIndex, stopIndex, findEntityDetails.getHeadline(), findEntityDetails.getSectionNumber(), findEntityDetails.isDictionaryEntry(), findEntityDetails.isDossierDictionary(), findEntityDetails.getEngine(), findEntityDetails.getEntityType())); } } + private boolean isSeparator(char c) { + + return Character.isWhitespace(c) || Pattern.matches("\\p{Punct}", String.valueOf(c)) || c == '\"' || c == '‘' || c == '’'; + } + public Set clearAndFindPositions(Set entities, SearchableText text, Dictionary dictionary) { @@ -110,7 +114,8 @@ public class EntitySearchUtils { .length() && inner.getStart() >= word.getStart() && inner.getEnd() <= word.getEnd() && word != inner && word.getSectionNumber() == inner.getSectionNumber()) { if (word.getEntityType().equals(EntityType.RECOMMENDATION) && inner.getEntityType().equals(EntityType.ENTITY)) { wordsToRemove.add(word); - } else { + } else if(!(inner.getEntityType() == EntityType.FALSE_RECOMMENDATION && word.getEntityType() == EntityType.ENTITY || + inner.getEntityType() == EntityType.ENTITY && word.getEntityType() == EntityType.FALSE_RECOMMENDATION)) { wordsToRemove.add(inner); } }