Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

Pull request #479: RED-5139: Bugfix if false positives are added, for rule added entries

Browse Source 
Merge in RED/redaction-service from kbudisantoso/RED-5139-backport to release/3.163.x

* commit '90b7875c0b00298cb6972ee2fed81cad63dfcf08':
  RED-5139: Bugfix if false positives are added, for rule added entries
This commit is contained in:
Kresnadi Budisantoso 2022-09-28 09:24:30 +02:00
commit f42537ea1a
2 changed files with 25 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java
View File

@ -863,6 +863,9 @@ public class Section {
}
private Set<Entity> findEntities(String value, String asType, boolean caseInsensitive, boolean redacted,
int ruleNumber, String reason, String legalBasis, Engine engine,
boolean asRecommendation) {
@ -935,22 +938,25 @@ public class Section {
singleEntitySet.add(entity);
EntitySearchUtils.clearAndFindPositions(singleEntitySet, searchableText, dictionary, manualRedactions);
EntitySearchUtils.removeFalsePositives(singleEntitySet, searchText, dictionary.getType(type), new FindEntityDetails(type, headline, sectionNumber, false, false, Engine.RULE, EntityType.ENTITY));
EntitySearchUtils.addEntitiesWithHigherRank(entities, entity, dictionary);
if (!singleEntitySet.isEmpty()) {
EntitySearchUtils.addEntitiesWithHigherRank(entities, singleEntitySet.iterator().next(), dictionary);
EntitySearchUtils.removeEntitiesContainedInLarger(entities);
EntitySearchUtils.removeEntitiesContainedInLarger(entities);
if (addAsRecommendations && !isLocal()) {
String cleanedWord = word.replaceAll(",", " ").replaceAll(" ", " ").trim() + " ";
Pattern pattern = Patterns.AUTHOR_TABLE_SPITTER;
Matcher matcher = pattern.matcher(cleanedWord);
if (addAsRecommendations && !isLocal()) {
String cleanedWord = word.replaceAll(",", " ").replaceAll(" ", " ").trim() + " ";
Pattern pattern = Patterns.AUTHOR_TABLE_SPITTER;
Matcher matcher = pattern.matcher(cleanedWord);
while (matcher.find()) {
String match = matcher.group().trim();
if (match.length() >= 3) {
localDictionaryAdds.computeIfAbsent(type, (x) -> new HashSet<>()).add(match);
String lastname = match.split(" ")[0];
localDictionaryAdds.computeIfAbsent(type, (x) -> new HashSet<>()).add(lastname);
while (matcher.find()) {
String match = matcher.group().trim();
if (match.length() >= 3) {
localDictionaryAdds.computeIfAbsent(type, (x) -> new HashSet<>()).add(match);
String lastname = match.split(" ")[0];
localDictionaryAdds.computeIfAbsent(type, (x) -> new HashSet<>()).add(lastname);
}
}
}
}

7
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java
View File

@ -20,6 +20,13 @@ public class EntitySearchUtils {
return searchImplementation.atLeastOneMatches(sectionText);
}
public void removeFalsePositives(Set<Entity> found, String inputString, DictionaryModel type, FindEntityDetails details){
Set<Entity> falsePositives = find(inputString, type.getFalsePositiveSearch(), details.withEntityType(EntityType.FALSE_POSITIVE));
markFalsePositives(found, falsePositives);
found.removeIf(f -> f.isFalsePositive());
}
public Set<Entity> findEntities(String inputString, SearchImplementation searchImplementation, DictionaryModel type, FindEntityDetails details) {
Set<Entity> found = find(inputString, searchImplementation, details);