From ddd4c42da7a43ca44e1eb23d2be72bcafdaefe3a Mon Sep 17 00:00:00 2001 From: Thomas Beyer Date: Fri, 24 Mar 2023 13:26:04 +0100 Subject: [PATCH 1/3] RED-6411 - remove false positives before calling EntitySearchUtils.addOrAddEngine() --- .../v1/server/redaction/service/EntityRedactionService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java index 55a6f1a2..8b49c3c4 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java @@ -300,7 +300,7 @@ public class EntityRedactionService { !local, model.isDossierDictionary(), local ? Engine.RULE : Engine.DICTIONARY, - local ? EntityType.RECOMMENDATION : EntityType.ENTITY)); + local ? EntityType.RECOMMENDATION : EntityType.ENTITY)).stream().filter(e -> !e.isFalsePositive()).collect(Collectors.toSet()); EntitySearchUtils.addOrAddEngine(found, entities); } From 8409b61f405ba0d942b5dcadeb866674a07c1838 Mon Sep 17 00:00:00 2001 From: Thomas Beyer Date: Mon, 27 Mar 2023 12:48:19 +0200 Subject: [PATCH 2/3] RED-6411 - extend logic when replacing entities with higher rank --- .../v1/server/redaction/utils/EntitySearchUtils.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java index 510e2276..097bc43c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java @@ -273,14 +273,14 @@ public final class EntitySearchUtils { existing.setLegalBasis(found.getLegalBasis()); existing.setMatchedRule(found.getMatchedRule()); existing.setRedactionReason(found.getRedactionReason()); - if (existing.getEntityType().equals(EntityType.RECOMMENDATION) && found.getEntityType().equals(EntityType.ENTITY) || existing.getEntityType() - .equals(EntityType.ENTITY) && found.getEntityType().equals(EntityType.RECOMMENDATION)) { + if (isOneARecommendationAndTheOtherEntity(found, existing)) { existing.setEntityType(EntityType.ENTITY); if (found.isRedaction()) { existing.setRedaction(true); } } - } else if (dictionary.getDictionaryRank(existing.getType()) <= dictionary.getDictionaryRank(found.getType())) { + } else if (dictionary.getDictionaryRank(existing.getType()) <= dictionary.getDictionaryRank(found.getType()) && + !(isOneARecommendationAndTheOtherEntity(found, existing) && existing.isRedaction() && found.isRedaction()) ) { entities.remove(found); entities.add(found); } @@ -289,7 +289,11 @@ public final class EntitySearchUtils { } } + private boolean isOneARecommendationAndTheOtherEntity (Entity found, Entity existing) { + return existing.getEntityType().equals(EntityType.RECOMMENDATION) && found.getEntityType().equals(EntityType.ENTITY) || existing.getEntityType() + .equals(EntityType.ENTITY) && found.getEntityType().equals(EntityType.RECOMMENDATION); + } public void addEntitiesIgnoreRank(Set entities, Set found) { // HashSet keeps old value but we want the new. entities.removeAll(found); From c7643da4788b7c9fb97428c409ff6e3eb2e19919 Mon Sep 17 00:00:00 2001 From: Thomas Beyer Date: Tue, 28 Mar 2023 11:23:52 +0200 Subject: [PATCH 3/3] RED-6411 - optimize method --- .../v1/server/redaction/utils/EntitySearchUtils.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java index 097bc43c..efbfcdc0 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java @@ -289,11 +289,14 @@ public final class EntitySearchUtils { } } - private boolean isOneARecommendationAndTheOtherEntity (Entity found, Entity existing) { + private boolean isOneARecommendationAndTheOtherEntity(Entity entityOne, Entity entityTwo) { - return existing.getEntityType().equals(EntityType.RECOMMENDATION) && found.getEntityType().equals(EntityType.ENTITY) || existing.getEntityType() - .equals(EntityType.ENTITY) && found.getEntityType().equals(EntityType.RECOMMENDATION); + var entityTypeOne = entityOne.getEntityType(); + var entityTypeTwo = entityTwo.getEntityType(); + return entityTypeTwo.equals(EntityType.RECOMMENDATION) && entityTypeOne.equals(EntityType.ENTITY) + || entityTypeTwo.equals(EntityType.ENTITY) && entityTypeOne.equals(EntityType.RECOMMENDATION); } + public void addEntitiesIgnoreRank(Set entities, Set found) { // HashSet keeps old value but we want the new. entities.removeAll(found);