Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-5139: Bugfix if false positives are added, for rule added entries

Browse Source
This commit is contained in:
deiflaender 2022-09-26 14:02:08 +02:00 committed by Kresnadi Budisantoso
parent de841452ba
commit 90b7875c0b
2 changed files with 25 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java
View File

@ -863,6 +863,9 @@ public class Section {
} }
private Set<Entity> findEntities(String value, String asType, boolean caseInsensitive, boolean redacted, private Set<Entity> findEntities(String value, String asType, boolean caseInsensitive, boolean redacted,
int ruleNumber, String reason, String legalBasis, Engine engine, int ruleNumber, String reason, String legalBasis, Engine engine,
boolean asRecommendation) { boolean asRecommendation) {
@ -935,22 +938,25 @@ public class Section {
singleEntitySet.add(entity); singleEntitySet.add(entity);
EntitySearchUtils.clearAndFindPositions(singleEntitySet, searchableText, dictionary, manualRedactions); EntitySearchUtils.clearAndFindPositions(singleEntitySet, searchableText, dictionary, manualRedactions);
EntitySearchUtils.removeFalsePositives(singleEntitySet, searchText, dictionary.getType(type), new FindEntityDetails(type, headline, sectionNumber, false, false, Engine.RULE, EntityType.ENTITY));
EntitySearchUtils.addEntitiesWithHigherRank(entities, entity, dictionary); if (!singleEntitySet.isEmpty()) {
EntitySearchUtils.addEntitiesWithHigherRank(entities, singleEntitySet.iterator().next(), dictionary);
EntitySearchUtils.removeEntitiesContainedInLarger(entities); EntitySearchUtils.removeEntitiesContainedInLarger(entities);
if (addAsRecommendations && !isLocal()) { if (addAsRecommendations && !isLocal()) {
String cleanedWord = word.replaceAll(",", " ").replaceAll(" ", " ").trim() + " "; String cleanedWord = word.replaceAll(",", " ").replaceAll(" ", " ").trim() + " ";
Pattern pattern = Patterns.AUTHOR_TABLE_SPITTER; Pattern pattern = Patterns.AUTHOR_TABLE_SPITTER;
Matcher matcher = pattern.matcher(cleanedWord); Matcher matcher = pattern.matcher(cleanedWord);
while (matcher.find()) { while (matcher.find()) {
String match = matcher.group().trim(); String match = matcher.group().trim();
if (match.length() >= 3) { if (match.length() >= 3) {
localDictionaryAdds.computeIfAbsent(type, (x) -> new HashSet<>()).add(match); localDictionaryAdds.computeIfAbsent(type, (x) -> new HashSet<>()).add(match);
String lastname = match.split(" ")[0]; String lastname = match.split(" ")[0];
localDictionaryAdds.computeIfAbsent(type, (x) -> new HashSet<>()).add(lastname); localDictionaryAdds.computeIfAbsent(type, (x) -> new HashSet<>()).add(lastname);
}
} }
} }
} }

7
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java
View File

@ -20,6 +20,13 @@ public class EntitySearchUtils {
return searchImplementation.atLeastOneMatches(sectionText); return searchImplementation.atLeastOneMatches(sectionText);
} }
public void removeFalsePositives(Set<Entity> found, String inputString, DictionaryModel type, FindEntityDetails details){
Set<Entity> falsePositives = find(inputString, type.getFalsePositiveSearch(), details.withEntityType(EntityType.FALSE_POSITIVE));
markFalsePositives(found, falsePositives);
found.removeIf(f -> f.isFalsePositive());
}
public Set<Entity> findEntities(String inputString, SearchImplementation searchImplementation, DictionaryModel type, FindEntityDetails details) { public Set<Entity> findEntities(String inputString, SearchImplementation searchImplementation, DictionaryModel type, FindEntityDetails details) {
Set<Entity> found = find(inputString, searchImplementation, details); Set<Entity> found = find(inputString, searchImplementation, details);