diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java index e364ec34..f1540ace 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java @@ -1,26 +1,34 @@ package com.iqser.red.service.redaction.v1.server.redaction.model; +import static com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary.RECOMMENDATION_PREFIX; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +import org.apache.commons.lang3.StringUtils; + import com.iqser.red.service.redaction.v1.model.ArgumentType; import com.iqser.red.service.redaction.v1.model.Engine; import com.iqser.red.service.redaction.v1.model.FileAttribute; import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock; import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils; import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns; + import lombok.Builder; import lombok.Data; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; - -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; -import java.util.*; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import java.util.stream.Collectors; - -import static com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary.RECOMMENDATION_PREFIX; @Data @Slf4j @@ -62,39 +70,69 @@ public class Section { @WhenCondition public boolean fileAttributeByIdEquals(@Argument(ArgumentType.FILE_ATTRIBUTE) String id, @Argument(ArgumentType.STRING) String value) { - return fileAttributes != null && fileAttributes.stream().filter(attribute -> id.equals(attribute.getId()) && value.equals(attribute.getValue())).findFirst().isPresent(); + + return fileAttributes != null && fileAttributes.stream() + .filter(attribute -> id.equals(attribute.getId()) && value.equals(attribute.getValue())) + .findFirst() + .isPresent(); } + @WhenCondition public boolean fileAttributeByPlaceholderEquals(@Argument(ArgumentType.FILE_ATTRIBUTE) String placeholder, @Argument(ArgumentType.STRING) String value) { - return fileAttributes != null && fileAttributes.stream().filter(attribute -> placeholder.equals(attribute.getPlaceholder()) && value.equals(attribute.getValue())).findFirst().isPresent(); + + return fileAttributes != null && fileAttributes.stream() + .filter(attribute -> placeholder.equals(attribute.getPlaceholder()) && value.equals(attribute.getValue())) + .findFirst() + .isPresent(); } + @WhenCondition public boolean fileAttributeByLabelEquals(@Argument(ArgumentType.FILE_ATTRIBUTE) String label, @Argument(ArgumentType.STRING) String value) { - return fileAttributes != null && fileAttributes.stream().filter(attribute -> label.equals(attribute.getLabel()) && value.equals(attribute.getValue())).findFirst().isPresent(); + + return fileAttributes != null && fileAttributes.stream() + .filter(attribute -> label.equals(attribute.getLabel()) && value.equals(attribute.getValue())) + .findFirst() + .isPresent(); } + @WhenCondition public boolean fileAttributeByIdEqualsIgnoreCase(@Argument(ArgumentType.FILE_ATTRIBUTE) String id, @Argument(ArgumentType.STRING) String value) { - return fileAttributes != null && fileAttributes.stream().filter(attribute -> id.equals(attribute.getId()) && value.equalsIgnoreCase(attribute.getValue())).findFirst().isPresent(); + + return fileAttributes != null && fileAttributes.stream() + .filter(attribute -> id.equals(attribute.getId()) && value.equalsIgnoreCase(attribute.getValue())) + .findFirst() + .isPresent(); } + @WhenCondition public boolean fileAttributeByPlaceholderEqualsIgnoreCase(@Argument(ArgumentType.FILE_ATTRIBUTE) String placeholder, @Argument(ArgumentType.STRING) String value) { - return fileAttributes != null && fileAttributes.stream().filter(attribute -> placeholder.equals(attribute.getPlaceholder()) && value.equalsIgnoreCase(attribute.getValue())).findFirst().isPresent(); + + return fileAttributes != null && fileAttributes.stream() + .filter(attribute -> placeholder.equals(attribute.getPlaceholder()) && value.equalsIgnoreCase(attribute.getValue())) + .findFirst() + .isPresent(); } + @WhenCondition public boolean fileAttributeByLabelEqualsIgnoreCase(@Argument(ArgumentType.FILE_ATTRIBUTE) String label, @Argument(ArgumentType.STRING) String value) { - return fileAttributes != null && fileAttributes.stream().filter(attribute -> label.equals(attribute.getLabel()) && value.equalsIgnoreCase(attribute.getValue())).findFirst().isPresent(); + + return fileAttributes != null && fileAttributes.stream() + .filter(attribute -> label.equals(attribute.getLabel()) && value.equalsIgnoreCase(attribute.getValue())) + .findFirst() + .isPresent(); } + @WhenCondition public boolean rowEquals(@Argument(ArgumentType.STRING) String headerName, @Argument(ArgumentType.STRING) String value) { @@ -106,6 +144,7 @@ public class Section { .equals(value); } + @WhenCondition public boolean hasTableHeader(@Argument(ArgumentType.STRING) String headerName) { @@ -113,18 +152,21 @@ public class Section { return tabularData != null && tabularData.containsKey(cleanHeaderName); } + @WhenCondition public boolean matchesType(@Argument(ArgumentType.TYPE) String type) { return entities.stream().anyMatch(entity -> entity.getType().equals(type)); } + @WhenCondition public boolean matchesImageType(@Argument(ArgumentType.TYPE) String type) { return images.stream().anyMatch(image -> image.getType().equals(type)); } + @WhenCondition public boolean headlineContainsWord(@Argument(ArgumentType.STRING) String word) { @@ -133,16 +175,16 @@ public class Section { @ThenAction - public void expandByRegEx(@Argument(ArgumentType.TYPE) String type, - @Argument(ArgumentType.REGEX) String pattern, + public void expandByRegEx(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.REGEX) String pattern, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, @Argument(ArgumentType.INTEGER) int group) { + expandByRegEx(type, pattern, patternCaseInsensitive, group, null); } + @ThenAction - public void expandByRegEx(@Argument(ArgumentType.TYPE) String type, - @Argument(ArgumentType.REGEX) String pattern, + public void expandByRegEx(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.REGEX) String pattern, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, @Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.REGEX) String withoutPattern) { @@ -162,7 +204,7 @@ public class Section { continue; } - if(withoutPattern != null) { + if (withoutPattern != null) { Matcher matcherWithout = compiledWithoutPattern.matcher(entity.getWord()); if (matcherWithout.find()) { continue; @@ -173,10 +215,9 @@ public class Section { while (matcher.find()) { String match = matcher.group(group); - if (StringUtils.isNotBlank(match)) { - expanded.addAll(findEntities(entity.getWord() + match, type, false, entity.isRedaction(), entity.getMatchedRule(), entity - .getRedactionReason(), entity.getLegalBasis())); + if (!overlapsMatchEntities(entities, match)) { + expanded.addAll(findEntities(entity.getWord() + match, type, false, entity.isRedaction(), entity.getMatchedRule(), entity.getRedactionReason(), entity.getLegalBasis())); } } } @@ -185,6 +226,20 @@ public class Section { EntitySearchUtils.removeEntitiesContainedInLarger(entities); } + + private boolean overlapsMatchEntities(Set entities, String match) { + + if (entities != null && StringUtils.isNotBlank(match)) { + for (Entity entity : entities) { + if (StringUtils.containsIgnoreCase(entity.getWord(), match.trim()) && entity.getEnd() + 1 >= entity.getStart()) { + return true; + } + } + } + return false; + } + + @ThenAction public void redactImage(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @@ -201,9 +256,9 @@ public class Section { }); } + @ThenAction - public void redact(@Argument(ArgumentType.TYPE) String type, - @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, + public void redact(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { @@ -220,6 +275,7 @@ public class Section { }); } + @ThenAction public void redactNotImage(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @@ -234,9 +290,9 @@ public class Section { }); } + @ThenAction - public void redactNot(@Argument(ArgumentType.TYPE) String type, - @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, + public void redactNot(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason) { boolean hasRecommendationDictionary = dictionaryTypes.contains(RECOMMENDATION_PREFIX + type); @@ -255,12 +311,14 @@ public class Section { @ThenAction public void redactNotAndReference(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.REFERENCE_TYPE) String referenceType, - @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, - @Argument(ArgumentType.STRING) String reason) { + @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, + @Argument(ArgumentType.STRING) String reason) { boolean hasRecommendationDictionary = dictionaryTypes.contains(RECOMMENDATION_PREFIX + type); - Set references = entities.stream().filter(entity -> entity.getType().equals(referenceType)).collect(Collectors.toSet()); + Set references = entities.stream() + .filter(entity -> entity.getType().equals(referenceType)) + .collect(Collectors.toSet()); entities.forEach(entity -> { if (entity.getType().equals(type) || hasRecommendationDictionary && entity.getType() @@ -274,7 +332,6 @@ public class Section { } - @ThenAction public void expandToHintAnnotationByRegEx(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.STRING) String pattern, @@ -305,6 +362,7 @@ public class Section { EntitySearchUtils.removeEntitiesContainedInLarger(entities); } + @ThenAction public void addHintAnnotationByRegEx(@Argument(ArgumentType.REGEX) String pattern, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, @@ -324,6 +382,7 @@ public class Section { } } + @ThenAction public void redactIfPrecededBy(@Argument(ArgumentType.STRING) String prefix, @Argument(ArgumentType.TYPE) String type, @@ -341,6 +400,7 @@ public class Section { }); } + @ThenAction public void addHintAnnotation(@Argument(ArgumentType.STRING) String value, @Argument(ArgumentType.TYPE) String asType) { @@ -349,9 +409,9 @@ public class Section { EntitySearchUtils.addEntitiesIgnoreRank(entities, found); } + @ThenAction - public void addRedaction(@Argument(ArgumentType.STRING) String value, - @Argument(ArgumentType.TYPE) String asType, + public void addRedaction(@Argument(ArgumentType.STRING) String value, @Argument(ArgumentType.TYPE) String asType, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { @@ -360,9 +420,9 @@ public class Section { EntitySearchUtils.addEntitiesIgnoreRank(entities, found); } + @ThenAction - public void redactLineAfter(@Argument(ArgumentType.STRING) String start, - @Argument(ArgumentType.TYPE) String asType, + public void redactLineAfter(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.TYPE) String asType, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere, @Argument(ArgumentType.STRING) String reason, @@ -384,6 +444,7 @@ public class Section { } } + @ThenAction public void recommendLineAfter(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.TYPE) String asType) { @@ -409,11 +470,11 @@ public class Section { } } + @ThenAction public void redactByRegEx(@Argument(ArgumentType.REGEX) String pattern, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, - @Argument(ArgumentType.INTEGER) int group, - @Argument(ArgumentType.TYPE) String asType, + @Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.TYPE) String asType, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { @@ -431,6 +492,7 @@ public class Section { } } + @ThenAction public void addRecommendationByRegEx(@Argument(ArgumentType.REGEX) String pattern, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, @@ -449,6 +511,7 @@ public class Section { } } + @ThenAction public void redactAndRecommendByRegEx(@Argument(ArgumentType.REGEX) String pattern, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, @@ -471,9 +534,9 @@ public class Section { } } + @ThenAction - public void redactBetween(@Argument(ArgumentType.STRING) String start, - @Argument(ArgumentType.STRING) String stop, + public void redactBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere, @@ -497,6 +560,7 @@ public class Section { } } + @ThenAction public void redactLinesBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop, @@ -531,6 +595,7 @@ public class Section { } } + @ThenAction public void highlightCell(@Argument(ArgumentType.STRING) String cellHeader, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @@ -539,10 +604,10 @@ public class Section { annotateCell(cellHeader, ruleNumber, type, false, false, null, null); } + @ThenAction public void redactCell(@Argument(ArgumentType.STRING) String cellHeader, - @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, - @Argument(ArgumentType.TYPE) String type, + @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.BOOLEAN) boolean addAsRecommendations, @Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { @@ -550,6 +615,7 @@ public class Section { annotateCell(cellHeader, ruleNumber, type, true, addAsRecommendations, reason, legalBasis); } + @ThenAction public void redactNotCell(@Argument(ArgumentType.STRING) String cellHeader, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @@ -636,6 +702,7 @@ public class Section { } } + @Retention(RetentionPolicy.RUNTIME) @Target(ElementType.METHOD) public @interface WhenCondition { @@ -653,6 +720,7 @@ public class Section { public @interface Argument { ArgumentType value() default ArgumentType.STRING; + } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/CBI_author.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/CBI_author.txt index e2854b4c..60ef872b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/CBI_author.txt +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/CBI_author.txt @@ -1,3 +1,5 @@ +Foo +F. Bar Johnson R | Weissler M S and Butters C A AD Hurt diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl index c0b96ff9..e4ef5964 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl @@ -19,8 +19,8 @@ rule "0: Expand CBI Authors with firstname initials" when Section(matchesType("CBI_author") || matchesType("recommendation_CBI_author")) then - section.expandByRegEx("CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1, "[^\\s]+"); - section.expandByRegEx("recommendation_CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1, "[^\\s]+"); + section.expandByRegEx("CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1, "[\\s]+"); + section.expandByRegEx("recommendation_CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1, "[\\s]+"); end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/Minimal Examples/foo.pdf b/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/Minimal Examples/foo.pdf new file mode 100644 index 00000000..b6ddcbc0 Binary files /dev/null and b/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/Minimal Examples/foo.pdf differ