Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-4799

Browse Source
This commit is contained in:
Timo Bejan 2022-07-27 12:27:20 +03:00
parent 05834865ec
commit bde4f22f3e
1 changed files with 24 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java
View File

@ -1,30 +1,15 @@
package com.iqser.red.service.redaction.v1.server.redaction.utils; package com.iqser.red.service.redaction.v1.server.redaction.utils;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import com.iqser.red.service.redaction.v1.model.Engine; import com.iqser.red.service.redaction.v1.model.Engine;
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary; import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrementValue; import com.iqser.red.service.redaction.v1.server.redaction.model.*;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
import lombok.experimental.UtilityClass; import lombok.experimental.UtilityClass;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import java.util.*;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
@Slf4j @Slf4j
@UtilityClass @UtilityClass
@SuppressWarnings("PMD") @SuppressWarnings("PMD")
@ -62,7 +47,7 @@ public class EntitySearchUtils {
Set<Entity> found = find(inputString, values, type.getType(), headline, sectionNumber, isDictionaryEntry, isDossierDictionary, engine, ignoreMinLength, asRecommendation ? EntityType.RECOMMENDATION : EntityType.ENTITY); Set<Entity> found = find(inputString, values, type.getType(), headline, sectionNumber, isDictionaryEntry, isDossierDictionary, engine, ignoreMinLength, asRecommendation ? EntityType.RECOMMENDATION : EntityType.ENTITY);
if(asRecommendation){ if (asRecommendation) {
Set<Entity> falseRecommendations = find(inputString, type.getFalseRecommendationValues(), type.getType(), headline, sectionNumber, isDictionaryEntry, isDossierDictionary, engine, ignoreMinLength, EntityType.FALSE_RECOMMENDATION); Set<Entity> falseRecommendations = find(inputString, type.getFalseRecommendationValues(), type.getType(), headline, sectionNumber, isDictionaryEntry, isDossierDictionary, engine, ignoreMinLength, EntityType.FALSE_RECOMMENDATION);
removeFalsePositives(found, falseRecommendations); removeFalsePositives(found, falseRecommendations);
found.addAll(falseRecommendations); found.addAll(falseRecommendations);
@ -103,7 +88,6 @@ public class EntitySearchUtils {
} }
private boolean isSeparator(char c) { private boolean isSeparator(char c) {
return Character.isWhitespace(c) || Pattern.matches("\\p{Punct}", String.valueOf(c)) || c == '\"' || c == '‘' || c == '’'; return Character.isWhitespace(c) || Pattern.matches("\\p{Punct}", String.valueOf(c)) || c == '\"' || c == '‘' || c == '’';
@ -140,13 +124,13 @@ public class EntitySearchUtils {
} }
public void removeFalsePositives(Set<Entity> entities, Set<Entity> falsePositives) { public void removeFalsePositives(Set<Entity> entities, Set<Entity> falsePositives) {
List<Entity> wordsToRemove = new ArrayList<>(); List<Entity> wordsToRemove = new ArrayList<>();
for (Entity word : falsePositives) { for (Entity word : falsePositives) {
for (Entity inner : entities) { for (Entity inner : entities) {
if (inner.getWord().length() < word.getWord() if (inner.getType().equals(word.getType()) &&
inner.getWord().length() < word.getWord()
.length() && inner.getStart() >= word.getStart() && inner.getEnd() <= word.getEnd() && word != inner && word.getSectionNumber() == inner.getSectionNumber()) { .length() && inner.getStart() >= word.getStart() && inner.getEnd() <= word.getEnd() && word != inner && word.getSectionNumber() == inner.getSectionNumber()) {
wordsToRemove.add(inner); wordsToRemove.add(inner);
} }
@ -157,8 +141,6 @@ public class EntitySearchUtils {
} }
public void removeEntitiesContainedInLarger(Set<Entity> entities) { public void removeEntitiesContainedInLarger(Set<Entity> entities) {
List<Entity> wordsToRemove = new ArrayList<>(); List<Entity> wordsToRemove = new ArrayList<>();
@ -226,10 +208,10 @@ public class EntitySearchUtils {
existing.setLegalBasis(found.getLegalBasis()); existing.setLegalBasis(found.getLegalBasis());
existing.setMatchedRule(found.getMatchedRule()); existing.setMatchedRule(found.getMatchedRule());
existing.setRedactionReason(found.getRedactionReason()); existing.setRedactionReason(found.getRedactionReason());
if(existing.getEntityType().equals(EntityType.RECOMMENDATION) && found.getEntityType().equals(EntityType.ENTITY) if (existing.getEntityType().equals(EntityType.RECOMMENDATION) && found.getEntityType().equals(EntityType.ENTITY)
|| existing.getEntityType().equals(EntityType.ENTITY) && found.getEntityType().equals(EntityType.RECOMMENDATION)){ || existing.getEntityType().equals(EntityType.ENTITY) && found.getEntityType().equals(EntityType.RECOMMENDATION)) {
existing.setEntityType(EntityType.ENTITY); existing.setEntityType(EntityType.ENTITY);
if(found.isRedaction()){ if (found.isRedaction()) {
existing.setRedaction(true); existing.setRedaction(true);
} }
} }