RED-5139: Bugfix if false positives are added, for rule added entries

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java

@@ -1002,10 +1002,15 @@ public class Section {
         }
     }
 
-
     private Set<Entity> findEntities(String value, String asType, boolean caseInsensitive, boolean redacted,
                                      int ruleNumber, String reason, String legalBasis, Engine engine,
                                      boolean asRecommendation) {
+        return findEntities(value, asType, caseInsensitive, redacted, true, ruleNumber, reason, legalBasis, engine, asRecommendation);
+    }
+
+    private Set<Entity> findEntities(String value, String asType, boolean caseInsensitive, boolean redacted, boolean excludeFalsePositives,
+                                     int ruleNumber, String reason, String legalBasis, Engine engine,
+                                     boolean asRecommendation) {
 
         String text = caseInsensitive ? searchText.toLowerCase() : searchText;
         Set<Entity> found = EntitySearchUtils.findEntities(text, new SearchImplementation(value, caseInsensitive), dictionary.getType(asType), new FindEntityDetails(asType, headline, sectionNumber, false, false, engine, asRecommendation ? EntityType.RECOMMENDATION : EntityType.ENTITY));
@@ -1019,9 +1024,14 @@ public class Section {
         });
 
         var cleared = EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary, manualRedactions);
+
+        if (excludeFalsePositives) {
             return cleared.stream().filter(f -> !f.isFalsePositive()).collect(Collectors.toSet());
         }
 
+        return cleared;
+    }
+
 
     private void redact(String type, int ruleNumber, String reason, String legalBasis, boolean redaction) {
 
@@ -1060,25 +1070,23 @@ public class Section {
         } else {
             String word = value.toString();
 
-            Entity entity = new Entity(word, type, value.getRowSpanStart(), value.getRowSpanStart() + word.length(), headline, sectionNumber, false, false, Engine.RULE, EntityType.ENTITY);
+            Set<Entity> found = findEntities(word.trim(), type, false, redact, false, ruleNumber, reason, legalBasis, Engine.RULE, false);
-            entity.setRedaction(redact);
+
-            entity.setMatchedRule(ruleNumber);
+            for (Entity entity: found) {// Should be found
-            entity.setRedactionReason(reason);
                 entity.setTargetSequences(value.getTextBlocks()
                         .stream()
                         .map(TextBlock::getSequences)
                         .flatMap(Collection::stream)
                         .collect(Collectors.toList())); // Make sure no other cells with same content are highlighted
-            entity.setLegalBasis(legalBasis);
 
-            Set<Entity> singleEntitySet = new HashSet<>();
+                if (entity.isFalsePositive()) {
-            singleEntitySet.add(entity);
+                    entity.setFalsePositive(true);
+                    entity.setRedaction(false);
+                }
+            }
 
-            EntitySearchUtils.clearAndFindPositions(singleEntitySet, searchableText, dictionary, manualRedactions);
+            EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary, manualRedactions);
-
+            EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
-            EntitySearchUtils.addEntitiesWithHigherRank(entities, entity, dictionary);
-
-            EntitySearchUtils.removeEntitiesContainedInLarger(entities);
 
             if (addAsRecommendations && !isLocal()) {
                 String cleanedWord = word.replaceAll(",", " ").replaceAll("  ", " ").trim() + " ";

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java

@@ -1,15 +1,29 @@
 package com.iqser.red.service.redaction.v1.server.redaction.utils;
 
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
-import com.iqser.red.service.redaction.v1.server.redaction.model.*;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
+import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
+import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
+import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
+
 import lombok.experimental.UtilityClass;
 import lombok.extern.slf4j.Slf4j;
 
-import java.util.*;
-import java.util.stream.Collectors;
-
 @Slf4j
 @UtilityClass
 @SuppressWarnings("PMD")
@@ -238,6 +252,11 @@ public class EntitySearchUtils {
                 existing.setLegalBasis(found.getLegalBasis());
                 existing.setMatchedRule(found.getMatchedRule());
                 existing.setRedactionReason(found.getRedactionReason());
+                existing.setFalsePositive(found.isFalsePositive());
+
+                if (found.isFalsePositive()) {
+                    existing.setRedaction(false);
+                }
                 if (existing.getEntityType().equals(EntityType.RECOMMENDATION) && found.getEntityType().equals(EntityType.ENTITY)
                         || existing.getEntityType().equals(EntityType.ENTITY) && found.getEntityType().equals(EntityType.RECOMMENDATION)) {
                     existing.setEntityType(EntityType.ENTITY);