Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-5139: Bugfix if false positives are added, for rule added entries

Browse Source
This commit is contained in:
Philipp Schramm 2022-09-26 10:09:55 +02:00
parent 142f5256ae
commit 4e96718ce3
2 changed files with 49 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java
View File

@ -1002,10 +1002,15 @@ public class Section {
}
}
private Set<Entity> findEntities(String value, String asType, boolean caseInsensitive, boolean redacted,
int ruleNumber, String reason, String legalBasis, Engine engine,
boolean asRecommendation) {
return findEntities(value, asType, caseInsensitive, redacted, true, ruleNumber, reason, legalBasis, engine, asRecommendation);
}
private Set<Entity> findEntities(String value, String asType, boolean caseInsensitive, boolean redacted, boolean excludeFalsePositives,
int ruleNumber, String reason, String legalBasis, Engine engine,
boolean asRecommendation) {
String text = caseInsensitive ? searchText.toLowerCase() : searchText;
Set<Entity> found = EntitySearchUtils.findEntities(text, new SearchImplementation(value, caseInsensitive), dictionary.getType(asType), new FindEntityDetails(asType, headline, sectionNumber, false, false, engine, asRecommendation ? EntityType.RECOMMENDATION : EntityType.ENTITY));
@ -1019,7 +1024,12 @@ public class Section {
});
var cleared = EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary, manualRedactions);
return cleared.stream().filter(f -> !f.isFalsePositive()).collect(Collectors.toSet());
if (excludeFalsePositives) {
return cleared.stream().filter(f -> !f.isFalsePositive()).collect(Collectors.toSet());
}
return cleared;
}
@ -1060,25 +1070,23 @@ public class Section {
} else {
String word = value.toString();
Entity entity = new Entity(word, type, value.getRowSpanStart(), value.getRowSpanStart() + word.length(), headline, sectionNumber, false, false, Engine.RULE, EntityType.ENTITY);
entity.setRedaction(redact);
entity.setMatchedRule(ruleNumber);
entity.setRedactionReason(reason);
entity.setTargetSequences(value.getTextBlocks()
.stream()
.map(TextBlock::getSequences)
.flatMap(Collection::stream)
.collect(Collectors.toList())); // Make sure no other cells with same content are highlighted
entity.setLegalBasis(legalBasis);
Set<Entity> found = findEntities(word.trim(), type, false, redact, false, ruleNumber, reason, legalBasis, Engine.RULE, false);
Set<Entity> singleEntitySet = new HashSet<>();
singleEntitySet.add(entity);
for (Entity entity: found) {// Should be found
entity.setTargetSequences(value.getTextBlocks()
.stream()
.map(TextBlock::getSequences)
.flatMap(Collection::stream)
.collect(Collectors.toList())); // Make sure no other cells with same content are highlighted
EntitySearchUtils.clearAndFindPositions(singleEntitySet, searchableText, dictionary, manualRedactions);
if (entity.isFalsePositive()) {
entity.setFalsePositive(true);
entity.setRedaction(false);
}
}
EntitySearchUtils.addEntitiesWithHigherRank(entities, entity, dictionary);
EntitySearchUtils.removeEntitiesContainedInLarger(entities);
EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary, manualRedactions);
EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
if (addAsRecommendations && !isLocal()) {
String cleanedWord = word.replaceAll(",", " ").replaceAll(" ", " ").trim() + " ";

27
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java
View File

@ -1,15 +1,29 @@
package com.iqser.red.service.redaction.v1.server.redaction.utils;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.*;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
import lombok.experimental.UtilityClass;
import lombok.extern.slf4j.Slf4j;
import java.util.*;
import java.util.stream.Collectors;
@Slf4j
@UtilityClass
@SuppressWarnings("PMD")
@ -238,6 +252,11 @@ public class EntitySearchUtils {
existing.setLegalBasis(found.getLegalBasis());
existing.setMatchedRule(found.getMatchedRule());
existing.setRedactionReason(found.getRedactionReason());
existing.setFalsePositive(found.isFalsePositive());
if (found.isFalsePositive()) {
existing.setRedaction(false);
}
if (existing.getEntityType().equals(EntityType.RECOMMENDATION) && found.getEntityType().equals(EntityType.ENTITY)
|| existing.getEntityType().equals(EntityType.ENTITY) && found.getEntityType().equals(EntityType.RECOMMENDATION)) {
existing.setEntityType(EntityType.ENTITY);