From bde4f22f3ef98fd44ce50ed422ed4ed1f90b9f0f Mon Sep 17 00:00:00 2001 From: Timo Bejan Date: Wed, 27 Jul 2022 12:27:20 +0300 Subject: [PATCH] RED-4799 --- .../redaction/utils/EntitySearchUtils.java | 66 +++++++------------ 1 file changed, 24 insertions(+), 42 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java index 7994e83a..9c41cef1 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java @@ -1,30 +1,15 @@ package com.iqser.red.service.redaction.v1.server.redaction.utils; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.regex.Pattern; -import java.util.stream.Collectors; - import com.iqser.red.service.redaction.v1.model.Engine; import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary; -import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrementValue; -import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel; -import com.iqser.red.service.redaction.v1.server.redaction.model.Entity; -import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence; -import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType; -import com.iqser.red.service.redaction.v1.server.redaction.model.Image; -import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText; - +import com.iqser.red.service.redaction.v1.server.redaction.model.*; import lombok.experimental.UtilityClass; import lombok.extern.slf4j.Slf4j; +import java.util.*; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + @Slf4j @UtilityClass @SuppressWarnings("PMD") @@ -60,24 +45,24 @@ public class EntitySearchUtils { public Set findEntities(String inputString, Set values, DictionaryModel type, String headline, int sectionNumber, boolean isDictionaryEntry, boolean isDossierDictionary, Engine engine, boolean ignoreMinLength, boolean asRecommendation) { - Set found = find(inputString, values, type.getType(), headline, sectionNumber, isDictionaryEntry, isDossierDictionary, engine, ignoreMinLength, asRecommendation ? EntityType.RECOMMENDATION : EntityType.ENTITY); + Set found = find(inputString, values, type.getType(), headline, sectionNumber, isDictionaryEntry, isDossierDictionary, engine, ignoreMinLength, asRecommendation ? EntityType.RECOMMENDATION : EntityType.ENTITY); - if(asRecommendation){ - Set falseRecommendations = find(inputString, type.getFalseRecommendationValues(), type.getType(), headline, sectionNumber, isDictionaryEntry, isDossierDictionary, engine, ignoreMinLength, EntityType.FALSE_RECOMMENDATION); - removeFalsePositives(found, falseRecommendations); - found.addAll(falseRecommendations); - } else { - Set falsePositives = find(inputString, type.getFalsePositiveValues(), type.getType(), headline, sectionNumber, isDictionaryEntry, isDossierDictionary, engine, ignoreMinLength, EntityType.FALSE_POSITIVE); - removeFalsePositives(found, falsePositives); - found.addAll(falsePositives); - } + if (asRecommendation) { + Set falseRecommendations = find(inputString, type.getFalseRecommendationValues(), type.getType(), headline, sectionNumber, isDictionaryEntry, isDossierDictionary, engine, ignoreMinLength, EntityType.FALSE_RECOMMENDATION); + removeFalsePositives(found, falseRecommendations); + found.addAll(falseRecommendations); + } else { + Set falsePositives = find(inputString, type.getFalsePositiveValues(), type.getType(), headline, sectionNumber, isDictionaryEntry, isDossierDictionary, engine, ignoreMinLength, EntityType.FALSE_POSITIVE); + removeFalsePositives(found, falsePositives); + found.addAll(falsePositives); + } - return found; + return found; } public Set find(String inputString, Set values, String type, String headline, int sectionNumber, boolean isDictionaryEntry, boolean isDossierDictionary, - Engine engine, boolean ignoreMinLength, EntityType entityType) { + Engine engine, boolean ignoreMinLength, EntityType entityType) { Set found = new HashSet<>(); for (String value : values) { @@ -103,7 +88,6 @@ public class EntitySearchUtils { } - private boolean isSeparator(char c) { return Character.isWhitespace(c) || Pattern.matches("\\p{Punct}", String.valueOf(c)) || c == '\"' || c == '‘' || c == '’'; @@ -140,15 +124,15 @@ public class EntitySearchUtils { } - public void removeFalsePositives(Set entities, Set falsePositives) { List wordsToRemove = new ArrayList<>(); for (Entity word : falsePositives) { for (Entity inner : entities) { - if (inner.getWord().length() < word.getWord() - .length() && inner.getStart() >= word.getStart() && inner.getEnd() <= word.getEnd() && word != inner && word.getSectionNumber() == inner.getSectionNumber()) { - wordsToRemove.add(inner); + if (inner.getType().equals(word.getType()) && + inner.getWord().length() < word.getWord() + .length() && inner.getStart() >= word.getStart() && inner.getEnd() <= word.getEnd() && word != inner && word.getSectionNumber() == inner.getSectionNumber()) { + wordsToRemove.add(inner); } } } @@ -157,8 +141,6 @@ public class EntitySearchUtils { } - - public void removeEntitiesContainedInLarger(Set entities) { List wordsToRemove = new ArrayList<>(); @@ -226,10 +208,10 @@ public class EntitySearchUtils { existing.setLegalBasis(found.getLegalBasis()); existing.setMatchedRule(found.getMatchedRule()); existing.setRedactionReason(found.getRedactionReason()); - if(existing.getEntityType().equals(EntityType.RECOMMENDATION) && found.getEntityType().equals(EntityType.ENTITY) - || existing.getEntityType().equals(EntityType.ENTITY) && found.getEntityType().equals(EntityType.RECOMMENDATION)){ + if (existing.getEntityType().equals(EntityType.RECOMMENDATION) && found.getEntityType().equals(EntityType.ENTITY) + || existing.getEntityType().equals(EntityType.ENTITY) && found.getEntityType().equals(EntityType.RECOMMENDATION)) { existing.setEntityType(EntityType.ENTITY); - if(found.isRedaction()){ + if (found.isRedaction()) { existing.setRedaction(true); } }