RED-2845 Bugfix: Avoid the expansion if it would result in a redaction overlap and bugfix in RegExp in drools

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java

@@ -1,26 +1,34 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model;
 
+import static com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary.RECOMMENDATION_PREFIX;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+import org.apache.commons.lang3.StringUtils;
+
 import com.iqser.red.service.redaction.v1.model.ArgumentType;
 import com.iqser.red.service.redaction.v1.model.Engine;
 import com.iqser.red.service.redaction.v1.model.FileAttribute;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns;
+
 import lombok.Builder;
 import lombok.Data;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.StringUtils;
-
-import java.lang.annotation.ElementType;
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
-import java.lang.annotation.Target;
-import java.util.*;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import java.util.stream.Collectors;
-
-import static com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary.RECOMMENDATION_PREFIX;
 
 @Data
 @Slf4j
@@ -62,39 +70,69 @@ public class Section {
     @WhenCondition
     public boolean fileAttributeByIdEquals(@Argument(ArgumentType.FILE_ATTRIBUTE) String id,
                                            @Argument(ArgumentType.STRING) String value) {
-        return fileAttributes != null && fileAttributes.stream().filter(attribute -> id.equals(attribute.getId()) && value.equals(attribute.getValue())).findFirst().isPresent();
+
+        return fileAttributes != null && fileAttributes.stream()
+                .filter(attribute -> id.equals(attribute.getId()) && value.equals(attribute.getValue()))
+                .findFirst()
+                .isPresent();
     }
 
+
     @WhenCondition
     public boolean fileAttributeByPlaceholderEquals(@Argument(ArgumentType.FILE_ATTRIBUTE) String placeholder,
                                                     @Argument(ArgumentType.STRING) String value) {
-        return fileAttributes != null && fileAttributes.stream().filter(attribute -> placeholder.equals(attribute.getPlaceholder()) && value.equals(attribute.getValue())).findFirst().isPresent();
+
+        return fileAttributes != null && fileAttributes.stream()
+                .filter(attribute -> placeholder.equals(attribute.getPlaceholder()) && value.equals(attribute.getValue()))
+                .findFirst()
+                .isPresent();
     }
 
+
     @WhenCondition
     public boolean fileAttributeByLabelEquals(@Argument(ArgumentType.FILE_ATTRIBUTE) String label,
                                               @Argument(ArgumentType.STRING) String value) {
-        return fileAttributes != null && fileAttributes.stream().filter(attribute -> label.equals(attribute.getLabel()) && value.equals(attribute.getValue())).findFirst().isPresent();
+
+        return fileAttributes != null && fileAttributes.stream()
+                .filter(attribute -> label.equals(attribute.getLabel()) && value.equals(attribute.getValue()))
+                .findFirst()
+                .isPresent();
     }
 
+
     @WhenCondition
     public boolean fileAttributeByIdEqualsIgnoreCase(@Argument(ArgumentType.FILE_ATTRIBUTE) String id,
                                                      @Argument(ArgumentType.STRING) String value) {
-        return fileAttributes != null && fileAttributes.stream().filter(attribute -> id.equals(attribute.getId()) && value.equalsIgnoreCase(attribute.getValue())).findFirst().isPresent();
+
+        return fileAttributes != null && fileAttributes.stream()
+                .filter(attribute -> id.equals(attribute.getId()) && value.equalsIgnoreCase(attribute.getValue()))
+                .findFirst()
+                .isPresent();
     }
 
+
     @WhenCondition
     public boolean fileAttributeByPlaceholderEqualsIgnoreCase(@Argument(ArgumentType.FILE_ATTRIBUTE) String placeholder,
                                                               @Argument(ArgumentType.STRING) String value) {
-        return fileAttributes != null && fileAttributes.stream().filter(attribute -> placeholder.equals(attribute.getPlaceholder()) && value.equalsIgnoreCase(attribute.getValue())).findFirst().isPresent();
+
+        return fileAttributes != null && fileAttributes.stream()
+                .filter(attribute -> placeholder.equals(attribute.getPlaceholder()) && value.equalsIgnoreCase(attribute.getValue()))
+                .findFirst()
+                .isPresent();
     }
 
+
     @WhenCondition
     public boolean fileAttributeByLabelEqualsIgnoreCase(@Argument(ArgumentType.FILE_ATTRIBUTE) String label,
                                                         @Argument(ArgumentType.STRING) String value) {
-        return fileAttributes != null && fileAttributes.stream().filter(attribute -> label.equals(attribute.getLabel()) && value.equalsIgnoreCase(attribute.getValue())).findFirst().isPresent();
+
+        return fileAttributes != null && fileAttributes.stream()
+                .filter(attribute -> label.equals(attribute.getLabel()) && value.equalsIgnoreCase(attribute.getValue()))
+                .findFirst()
+                .isPresent();
     }
 
+
     @WhenCondition
     public boolean rowEquals(@Argument(ArgumentType.STRING) String headerName,
                              @Argument(ArgumentType.STRING) String value) {
@@ -106,6 +144,7 @@ public class Section {
                 .equals(value);
     }
 
+
     @WhenCondition
     public boolean hasTableHeader(@Argument(ArgumentType.STRING) String headerName) {
 
@@ -113,18 +152,21 @@ public class Section {
         return tabularData != null && tabularData.containsKey(cleanHeaderName);
     }
 
+
     @WhenCondition
     public boolean matchesType(@Argument(ArgumentType.TYPE) String type) {
 
         return entities.stream().anyMatch(entity -> entity.getType().equals(type));
     }
 
+
     @WhenCondition
     public boolean matchesImageType(@Argument(ArgumentType.TYPE) String type) {
 
         return images.stream().anyMatch(image -> image.getType().equals(type));
     }
 
+
     @WhenCondition
     public boolean headlineContainsWord(@Argument(ArgumentType.STRING) String word) {
 
@@ -133,16 +175,16 @@ public class Section {
 
 
     @ThenAction
-    public void expandByRegEx(@Argument(ArgumentType.TYPE) String type,
+    public void expandByRegEx(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.REGEX) String pattern,
-                              @Argument(ArgumentType.REGEX) String pattern,
                               @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
                               @Argument(ArgumentType.INTEGER) int group) {
+
         expandByRegEx(type, pattern, patternCaseInsensitive, group, null);
     }
 
+
     @ThenAction
-    public void expandByRegEx(@Argument(ArgumentType.TYPE) String type,
+    public void expandByRegEx(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.REGEX) String pattern,
-                              @Argument(ArgumentType.REGEX) String pattern,
                               @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
                               @Argument(ArgumentType.INTEGER) int group,
                               @Argument(ArgumentType.REGEX) String withoutPattern) {
@@ -162,7 +204,7 @@ public class Section {
                 continue;
             }
 
-            if(withoutPattern != null) {
+            if (withoutPattern != null) {
                 Matcher matcherWithout = compiledWithoutPattern.matcher(entity.getWord());
                 if (matcherWithout.find()) {
                     continue;
@@ -173,10 +215,9 @@ public class Section {
 
             while (matcher.find()) {
                 String match = matcher.group(group);
-                if (StringUtils.isNotBlank(match)) {
-                    expanded.addAll(findEntities(entity.getWord() + match, type, false, entity.isRedaction(), entity.getMatchedRule(), entity
-                            .getRedactionReason(), entity.getLegalBasis()));
 
+                if (!overlapsMatchEntities(entities, match)) {
+                    expanded.addAll(findEntities(entity.getWord() + match, type, false, entity.isRedaction(), entity.getMatchedRule(), entity.getRedactionReason(), entity.getLegalBasis()));
                 }
             }
         }
@@ -185,6 +226,20 @@ public class Section {
         EntitySearchUtils.removeEntitiesContainedInLarger(entities);
     }
 
+
+    private boolean overlapsMatchEntities(Set<Entity> entities, String match) {
+
+        if (entities != null && StringUtils.isNotBlank(match)) {
+            for (Entity entity : entities) {
+                if (StringUtils.containsIgnoreCase(entity.getWord(), match.trim()) && entity.getEnd() + 1 >= entity.getStart()) {
+                    return true;
+                }
+            }
+        }
+        return false;
+    }
+
+
     @ThenAction
     public void redactImage(@Argument(ArgumentType.TYPE) String type,
                             @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
@@ -201,9 +256,9 @@ public class Section {
         });
     }
 
+
     @ThenAction
-    public void redact(@Argument(ArgumentType.TYPE) String type,
+    public void redact(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
-                       @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
                        @Argument(ArgumentType.STRING) String reason,
                        @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
 
@@ -220,6 +275,7 @@ public class Section {
         });
     }
 
+
     @ThenAction
     public void redactNotImage(@Argument(ArgumentType.TYPE) String type,
                                @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
@@ -234,9 +290,9 @@ public class Section {
         });
     }
 
+
     @ThenAction
-    public void redactNot(@Argument(ArgumentType.TYPE) String type,
+    public void redactNot(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
-                          @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
                           @Argument(ArgumentType.STRING) String reason) {
 
         boolean hasRecommendationDictionary = dictionaryTypes.contains(RECOMMENDATION_PREFIX + type);
@@ -260,7 +316,9 @@ public class Section {
 
         boolean hasRecommendationDictionary = dictionaryTypes.contains(RECOMMENDATION_PREFIX + type);
 
-        Set<Entity> references = entities.stream().filter(entity -> entity.getType().equals(referenceType)).collect(Collectors.toSet());
+        Set<Entity> references = entities.stream()
+                .filter(entity -> entity.getType().equals(referenceType))
+                .collect(Collectors.toSet());
 
         entities.forEach(entity -> {
             if (entity.getType().equals(type) || hasRecommendationDictionary && entity.getType()
@@ -274,7 +332,6 @@ public class Section {
     }
 
 
-
     @ThenAction
     public void expandToHintAnnotationByRegEx(@Argument(ArgumentType.TYPE) String type,
                                               @Argument(ArgumentType.STRING) String pattern,
@@ -305,6 +362,7 @@ public class Section {
         EntitySearchUtils.removeEntitiesContainedInLarger(entities);
     }
 
+
     @ThenAction
     public void addHintAnnotationByRegEx(@Argument(ArgumentType.REGEX) String pattern,
                                          @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
@@ -324,6 +382,7 @@ public class Section {
         }
     }
 
+
     @ThenAction
     public void redactIfPrecededBy(@Argument(ArgumentType.STRING) String prefix,
                                    @Argument(ArgumentType.TYPE) String type,
@@ -341,6 +400,7 @@ public class Section {
         });
     }
 
+
     @ThenAction
     public void addHintAnnotation(@Argument(ArgumentType.STRING) String value,
                                   @Argument(ArgumentType.TYPE) String asType) {
@@ -349,9 +409,9 @@ public class Section {
         EntitySearchUtils.addEntitiesIgnoreRank(entities, found);
     }
 
+
     @ThenAction
-    public void addRedaction(@Argument(ArgumentType.STRING) String value,
+    public void addRedaction(@Argument(ArgumentType.STRING) String value, @Argument(ArgumentType.TYPE) String asType,
-                             @Argument(ArgumentType.TYPE) String asType,
                              @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
                              @Argument(ArgumentType.STRING) String reason,
                              @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
@@ -360,9 +420,9 @@ public class Section {
         EntitySearchUtils.addEntitiesIgnoreRank(entities, found);
     }
 
+
     @ThenAction
-    public void redactLineAfter(@Argument(ArgumentType.STRING) String start,
+    public void redactLineAfter(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.TYPE) String asType,
-                                @Argument(ArgumentType.TYPE) String asType,
                                 @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
                                 @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
                                 @Argument(ArgumentType.STRING) String reason,
@@ -384,6 +444,7 @@ public class Section {
         }
     }
 
+
     @ThenAction
     public void recommendLineAfter(@Argument(ArgumentType.STRING) String start,
                                    @Argument(ArgumentType.TYPE) String asType) {
@@ -409,11 +470,11 @@ public class Section {
         }
     }
 
+
     @ThenAction
     public void redactByRegEx(@Argument(ArgumentType.REGEX) String pattern,
                               @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
-                              @Argument(ArgumentType.INTEGER) int group,
+                              @Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.TYPE) String asType,
-                              @Argument(ArgumentType.TYPE) String asType,
                               @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
                               @Argument(ArgumentType.STRING) String reason,
                               @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
@@ -431,6 +492,7 @@ public class Section {
         }
     }
 
+
     @ThenAction
     public void addRecommendationByRegEx(@Argument(ArgumentType.REGEX) String pattern,
                                          @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
@@ -449,6 +511,7 @@ public class Section {
         }
     }
 
+
     @ThenAction
     public void redactAndRecommendByRegEx(@Argument(ArgumentType.REGEX) String pattern,
                                           @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
@@ -471,9 +534,9 @@ public class Section {
         }
     }
 
+
     @ThenAction
-    public void redactBetween(@Argument(ArgumentType.STRING) String start,
+    public void redactBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop,
-                              @Argument(ArgumentType.STRING) String stop,
                               @Argument(ArgumentType.TYPE) String asType,
                               @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
                               @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
@@ -497,6 +560,7 @@ public class Section {
         }
     }
 
+
     @ThenAction
     public void redactLinesBetween(@Argument(ArgumentType.STRING) String start,
                                    @Argument(ArgumentType.STRING) String stop,
@@ -531,6 +595,7 @@ public class Section {
         }
     }
 
+
     @ThenAction
     public void highlightCell(@Argument(ArgumentType.STRING) String cellHeader,
                               @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
@@ -539,10 +604,10 @@ public class Section {
         annotateCell(cellHeader, ruleNumber, type, false, false, null, null);
     }
 
+
     @ThenAction
     public void redactCell(@Argument(ArgumentType.STRING) String cellHeader,
-                           @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
+                           @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.TYPE) String type,
-                           @Argument(ArgumentType.TYPE) String type,
                            @Argument(ArgumentType.BOOLEAN) boolean addAsRecommendations,
                            @Argument(ArgumentType.STRING) String reason,
                            @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
@@ -550,6 +615,7 @@ public class Section {
         annotateCell(cellHeader, ruleNumber, type, true, addAsRecommendations, reason, legalBasis);
     }
 
+
     @ThenAction
     public void redactNotCell(@Argument(ArgumentType.STRING) String cellHeader,
                               @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
@@ -636,6 +702,7 @@ public class Section {
         }
     }
 
+
     @Retention(RetentionPolicy.RUNTIME)
     @Target(ElementType.METHOD)
     public @interface WhenCondition {
@@ -653,6 +720,7 @@ public class Section {
     public @interface Argument {
 
         ArgumentType value() default ArgumentType.STRING;
+
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/CBI_author.txt

@@ -1,3 +1,5 @@
+Foo
+F. Bar
 Johnson R |
 Weissler M S and Butters C A
 AD Hurt

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl

@@ -19,8 +19,8 @@ rule "0: Expand CBI Authors with firstname initials"
     when
         Section(matchesType("CBI_author") || matchesType("recommendation_CBI_author"))
     then
-        section.expandByRegEx("CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1, "[^\\s]+");
+        section.expandByRegEx("CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1, "[\\s]+");
-        section.expandByRegEx("recommendation_CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1, "[^\\s]+");
+        section.expandByRegEx("recommendation_CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1, "[\\s]+");
     end