From 4e96718ce32982a206ddf941ab5c86e456d9ed8f Mon Sep 17 00:00:00 2001 From: Philipp Schramm Date: Mon, 26 Sep 2022 10:09:55 +0200 Subject: [PATCH] RED-5139: Bugfix if false positives are added, for rule added entries --- .../v1/server/redaction/model/Section.java | 44 +++++++++++-------- .../redaction/utils/EntitySearchUtils.java | 27 ++++++++++-- 2 files changed, 49 insertions(+), 22 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java index 779d8217..6961030f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java @@ -1002,10 +1002,15 @@ public class Section { } } - private Set findEntities(String value, String asType, boolean caseInsensitive, boolean redacted, int ruleNumber, String reason, String legalBasis, Engine engine, boolean asRecommendation) { + return findEntities(value, asType, caseInsensitive, redacted, true, ruleNumber, reason, legalBasis, engine, asRecommendation); + } + + private Set findEntities(String value, String asType, boolean caseInsensitive, boolean redacted, boolean excludeFalsePositives, + int ruleNumber, String reason, String legalBasis, Engine engine, + boolean asRecommendation) { String text = caseInsensitive ? searchText.toLowerCase() : searchText; Set found = EntitySearchUtils.findEntities(text, new SearchImplementation(value, caseInsensitive), dictionary.getType(asType), new FindEntityDetails(asType, headline, sectionNumber, false, false, engine, asRecommendation ? EntityType.RECOMMENDATION : EntityType.ENTITY)); @@ -1019,7 +1024,12 @@ public class Section { }); var cleared = EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary, manualRedactions); - return cleared.stream().filter(f -> !f.isFalsePositive()).collect(Collectors.toSet()); + + if (excludeFalsePositives) { + return cleared.stream().filter(f -> !f.isFalsePositive()).collect(Collectors.toSet()); + } + + return cleared; } @@ -1060,25 +1070,23 @@ public class Section { } else { String word = value.toString(); - Entity entity = new Entity(word, type, value.getRowSpanStart(), value.getRowSpanStart() + word.length(), headline, sectionNumber, false, false, Engine.RULE, EntityType.ENTITY); - entity.setRedaction(redact); - entity.setMatchedRule(ruleNumber); - entity.setRedactionReason(reason); - entity.setTargetSequences(value.getTextBlocks() - .stream() - .map(TextBlock::getSequences) - .flatMap(Collection::stream) - .collect(Collectors.toList())); // Make sure no other cells with same content are highlighted - entity.setLegalBasis(legalBasis); + Set found = findEntities(word.trim(), type, false, redact, false, ruleNumber, reason, legalBasis, Engine.RULE, false); - Set singleEntitySet = new HashSet<>(); - singleEntitySet.add(entity); + for (Entity entity: found) {// Should be found + entity.setTargetSequences(value.getTextBlocks() + .stream() + .map(TextBlock::getSequences) + .flatMap(Collection::stream) + .collect(Collectors.toList())); // Make sure no other cells with same content are highlighted - EntitySearchUtils.clearAndFindPositions(singleEntitySet, searchableText, dictionary, manualRedactions); + if (entity.isFalsePositive()) { + entity.setFalsePositive(true); + entity.setRedaction(false); + } + } - EntitySearchUtils.addEntitiesWithHigherRank(entities, entity, dictionary); - - EntitySearchUtils.removeEntitiesContainedInLarger(entities); + EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary, manualRedactions); + EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary); if (addAsRecommendations && !isLocal()) { String cleanedWord = word.replaceAll(",", " ").replaceAll(" ", " ").trim() + " "; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java index 80c7d57c..f90c26da 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java @@ -1,15 +1,29 @@ package com.iqser.red.service.redaction.v1.server.redaction.utils; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus; import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions; import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary; -import com.iqser.red.service.redaction.v1.server.redaction.model.*; +import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel; +import com.iqser.red.service.redaction.v1.server.redaction.model.Entity; +import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence; +import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType; +import com.iqser.red.service.redaction.v1.server.redaction.model.Image; +import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText; + import lombok.experimental.UtilityClass; import lombok.extern.slf4j.Slf4j; -import java.util.*; -import java.util.stream.Collectors; - @Slf4j @UtilityClass @SuppressWarnings("PMD") @@ -238,6 +252,11 @@ public class EntitySearchUtils { existing.setLegalBasis(found.getLegalBasis()); existing.setMatchedRule(found.getMatchedRule()); existing.setRedactionReason(found.getRedactionReason()); + existing.setFalsePositive(found.isFalsePositive()); + + if (found.isFalsePositive()) { + existing.setRedaction(false); + } if (existing.getEntityType().equals(EntityType.RECOMMENDATION) && found.getEntityType().equals(EntityType.ENTITY) || existing.getEntityType().equals(EntityType.ENTITY) && found.getEntityType().equals(EntityType.RECOMMENDATION)) { existing.setEntityType(EntityType.ENTITY);