From 97fe7383cd9737f25d22903a6124dfe8361f7a84 Mon Sep 17 00:00:00 2001 From: Philipp Schramm Date: Mon, 28 Feb 2022 11:32:35 +0100 Subject: [PATCH] RED-3403 Extended Section with negated Methods --- .../v1/server/redaction/model/Section.java | 679 +++++++++--------- 1 file changed, 330 insertions(+), 349 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java index 17a5fad6..9a9b4734 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java @@ -8,11 +8,9 @@ import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; @@ -74,9 +72,7 @@ public class Section { public void addAiEntities(String type, String asType) { - Set entitiesOfType = nerEntities.stream() - .filter(nerEntity -> nerEntity.getType().equals(type)) - .collect(Collectors.toSet()); + Set entitiesOfType = nerEntities.stream().filter(nerEntity -> nerEntity.getType().equals(type)).collect(Collectors.toSet()); Set values = entitiesOfType.stream().map(Entity::getWord).collect(Collectors.toSet()); Set found = EntitySearchUtils.find(searchText, values, asType, headline, sectionNumber, false, false, Engine.NER, true); EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary); @@ -84,13 +80,12 @@ public class Section { Set finalResult = new HashSet<>(); // Only keep Entities with correct offsets from AI Service. - Iterator itty = found.iterator(); - while (itty.hasNext()) { - Entity current = itty.next(); + for (Entity current : found) { boolean foundSameOffsets = false; for (Entity entity : entitiesOfType) { if (entity.getStart().equals(current.getStart()) && entity.getEnd().equals(current.getEnd())) { foundSameOffsets = true; + break; } } if (foundSameOffsets) { @@ -105,14 +100,11 @@ public class Section { } - public void combineAiTypes(String startType, String combineTypes, int maxDistanceBetween, String asType, - int minPartMatches, boolean allowDuplicateTypes) { + public void combineAiTypes(String startType, String combineTypes, int maxDistanceBetween, String asType, int minPartMatches, boolean allowDuplicateTypes) { Set combineSet = Set.of(combineTypes.split(",")); - List sorted = nerEntities.stream() - .sorted(Comparator.comparing(Entity::getStart)) - .collect(Collectors.toList()); + List sorted = nerEntities.stream().sorted(Comparator.comparing(Entity::getStart)).collect(Collectors.toList()); Set found = new HashSet<>(); int start = -1; int lastEnd = -1; @@ -171,80 +163,45 @@ public class Section { @WhenCondition - public boolean fileAttributeByIdEquals(@Argument(ArgumentType.FILE_ATTRIBUTE) String id, - @Argument(ArgumentType.STRING) String value) { + public boolean fileAttributeByIdEquals(@Argument(ArgumentType.FILE_ATTRIBUTE) String id, @Argument(ArgumentType.STRING) String value) { - return fileAttributes != null && fileAttributes.stream() - .filter(attribute -> id.equals(attribute.getId()) && value.equals(attribute.getValue())) - .findFirst() - .isPresent(); + return fileAttributes != null && fileAttributes.stream().anyMatch(attribute -> id.equals(attribute.getId()) && value.equals(attribute.getValue())); } @WhenCondition - public boolean fileAttributeByPlaceholderEquals(@Argument(ArgumentType.FILE_ATTRIBUTE) String placeholder, - @Argument(ArgumentType.STRING) String value) { + public boolean fileAttributeByPlaceholderEquals(@Argument(ArgumentType.FILE_ATTRIBUTE) String placeholder, @Argument(ArgumentType.STRING) String value) { - return fileAttributes != null && fileAttributes.stream() - .filter(attribute -> placeholder.equals(attribute.getPlaceholder()) && value.equals(attribute.getValue())) - .findFirst() - .isPresent(); + return fileAttributes != null && fileAttributes.stream().anyMatch(attribute -> placeholder.equals(attribute.getPlaceholder()) && value.equals(attribute.getValue())); } @WhenCondition - public boolean fileAttributeByLabelEquals(@Argument(ArgumentType.FILE_ATTRIBUTE) String label, - @Argument(ArgumentType.STRING) String value) { + public boolean fileAttributeByLabelEquals(@Argument(ArgumentType.FILE_ATTRIBUTE) String label, @Argument(ArgumentType.STRING) String value) { - return fileAttributes != null && fileAttributes.stream() - .filter(attribute -> label.equals(attribute.getLabel()) && value.equals(attribute.getValue())) - .findFirst() - .isPresent(); + return fileAttributes != null && fileAttributes.stream().anyMatch(attribute -> label.equals(attribute.getLabel()) && value.equals(attribute.getValue())); } @WhenCondition - public boolean fileAttributeByIdEqualsIgnoreCase(@Argument(ArgumentType.FILE_ATTRIBUTE) String id, - @Argument(ArgumentType.STRING) String value) { + public boolean fileAttributeByIdEqualsIgnoreCase(@Argument(ArgumentType.FILE_ATTRIBUTE) String id, @Argument(ArgumentType.STRING) String value) { - return fileAttributes != null && fileAttributes.stream() - .filter(attribute -> id.equals(attribute.getId()) && value.equalsIgnoreCase(attribute.getValue())) - .findFirst() - .isPresent(); + return fileAttributes != null && fileAttributes.stream().anyMatch(attribute -> id.equals(attribute.getId()) && value.equalsIgnoreCase(attribute.getValue())); } @WhenCondition - public boolean fileAttributeByPlaceholderEqualsIgnoreCase(@Argument(ArgumentType.FILE_ATTRIBUTE) String placeholder, - @Argument(ArgumentType.STRING) String value) { + public boolean fileAttributeByPlaceholderEqualsIgnoreCase(@Argument(ArgumentType.FILE_ATTRIBUTE) String placeholder, @Argument(ArgumentType.STRING) String value) { return fileAttributes != null && fileAttributes.stream() - .filter(attribute -> placeholder.equals(attribute.getPlaceholder()) && value.equalsIgnoreCase(attribute.getValue())) - .findFirst() - .isPresent(); + .anyMatch(attribute -> placeholder.equals(attribute.getPlaceholder()) && value.equalsIgnoreCase(attribute.getValue())); } @WhenCondition - public boolean fileAttributeByLabelEqualsIgnoreCase(@Argument(ArgumentType.FILE_ATTRIBUTE) String label, - @Argument(ArgumentType.STRING) String value) { + public boolean fileAttributeByLabelEqualsIgnoreCase(@Argument(ArgumentType.FILE_ATTRIBUTE) String label, @Argument(ArgumentType.STRING) String value) { - return fileAttributes != null && fileAttributes.stream() - .filter(attribute -> label.equals(attribute.getLabel()) && value.equalsIgnoreCase(attribute.getValue())) - .findFirst() - .isPresent(); - } - - - @WhenCondition - public boolean rowEquals(@Argument(ArgumentType.STRING) String headerName, - @Argument(ArgumentType.STRING) String value) { - - String cleanHeaderName = headerName.replaceAll("\n", "").replaceAll(" ", "").replaceAll("-", ""); - - return tabularData != null && tabularData.containsKey(cleanHeaderName) && tabularData.get(cleanHeaderName) - .toString() - .equals(value); + return fileAttributes != null && fileAttributes.stream().anyMatch(attribute -> label.equals(attribute.getLabel()) && value.equalsIgnoreCase(attribute.getValue())); } @@ -284,21 +241,26 @@ public class Section { } + @WhenCondition + public boolean rowEquals(@Argument(ArgumentType.STRING) String headerName, @Argument(ArgumentType.STRING) String value) { + + String cleanHeaderName = headerName.replaceAll("\n", "").replaceAll(" ", "").replaceAll("-", ""); + + return tabularData != null && tabularData.containsKey(cleanHeaderName) && tabularData.get(cleanHeaderName).toString().equals(value); + } + + @ThenAction - public void expandByRegEx(@Argument(ArgumentType.TYPE) String type, - @Argument(ArgumentType.REGEX) String suffixPattern, - @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, - @Argument(ArgumentType.INTEGER) int group) { + public void expandByRegEx(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.REGEX) String suffixPattern, + @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, @Argument(ArgumentType.INTEGER) int group) { expandByRegEx(type, suffixPattern, patternCaseInsensitive, group, null); } @ThenAction - public void expandByRegEx(@Argument(ArgumentType.TYPE) String type, - @Argument(ArgumentType.REGEX) String suffixPattern, - @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, - @Argument(ArgumentType.INTEGER) int group, + public void expandByRegEx(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.REGEX) String suffixPattern, + @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, @Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.REGEX) String valuePattern) { Pattern compiledValuePattern = null; @@ -341,94 +303,169 @@ public class Section { @ThenAction - public void redactImage(@Argument(ArgumentType.TYPE) String type, - @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, - @Argument(ArgumentType.STRING) String reason, + public void redactImage(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { - images.forEach(image -> { - if (image.getType().equals(type)) { - image.setRedaction(true); - image.setMatchedRule(ruleNumber); - image.setRedactionReason(reason); - image.setLegalBasis(legalBasis); - } - }); + redactImage(type, ruleNumber, reason, legalBasis, true); } @ThenAction - public void redact(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, - @Argument(ArgumentType.STRING) String reason, + public void redactNotImage(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason) { + + redactImage(type, ruleNumber, reason, null, false); + } + + + @ThenAction + public void redact(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { - boolean hasRecommendationDictionary = dictionaryTypes.contains(RECOMMENDATION_PREFIX + type); - - entities.forEach(entity -> { - if (entity.getType().equals(type) || hasRecommendationDictionary && entity.getType() - .equals(RECOMMENDATION_PREFIX + type)) { - entity.setRedaction(true); - entity.setMatchedRule(ruleNumber); - entity.setRedactionReason(reason); - entity.setLegalBasis(legalBasis); - } - }); + redact(type, ruleNumber, reason, legalBasis, true); } @ThenAction - public void redactNotImage(@Argument(ArgumentType.TYPE) String type, - @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, - @Argument(ArgumentType.STRING) String reason) { + public void redactNot(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason) { - images.forEach(image -> { - if (image.getType().equals(type)) { - image.setRedaction(false); - image.setMatchedRule(ruleNumber); - image.setRedactionReason(reason); - } - }); + redact(type, ruleNumber, reason, null, false); } @ThenAction - public void redactNot(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, - @Argument(ArgumentType.STRING) String reason) { + public void redactLineAfter(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.TYPE) String asType, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, + @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere, @Argument(ArgumentType.STRING) String reason, + @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { - boolean hasRecommendationDictionary = dictionaryTypes.contains(RECOMMENDATION_PREFIX + type); - - entities.forEach(entity -> { - if (entity.getType().equals(type) || hasRecommendationDictionary && entity.getType() - .equals(RECOMMENDATION_PREFIX + type)) { - entity.setRedaction(false); - entity.setMatchedRule(ruleNumber); - entity.setRedactionReason(reason); - } - }); - } - - - public void ignore(String type) { - - entities.removeIf(entity -> entity.getType().equals(type)); + redactLineAfter(start, asType, ruleNumber, redactEverywhere, reason, legalBasis, true); } @ThenAction - public void redactNotAndReference(@Argument(ArgumentType.TYPE) String type, - @Argument(ArgumentType.REFERENCE_TYPE) String referenceType, - @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, + public void redactNotLineAfter(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.TYPE) String asType, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, + @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere, @Argument(ArgumentType.STRING) String reason) { + + redactLineAfter(start, asType, ruleNumber, redactEverywhere, reason, null, false); + + } + + + @ThenAction + public void redactByRegEx(@Argument(ArgumentType.REGEX) String pattern, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, + @Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.TYPE) String asType, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, + @Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { + + redactByRegEx(pattern, patternCaseInsensitive, group, asType, ruleNumber, reason, legalBasis, true); + } + + + @ThenAction + public void redactNotByRegEx(@Argument(ArgumentType.REGEX) String pattern, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, + @Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.TYPE) String asType, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, + @Argument(ArgumentType.STRING) String reason) { + + redactByRegEx(pattern, patternCaseInsensitive, group, asType, ruleNumber, reason, null, false); + } + + + @ThenAction + public void redactBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType, + @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere, + @Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { + + redactBetween(start, stop, asType, ruleNumber, redactEverywhere, reason, legalBasis, true); + } + + + @ThenAction + public void redactNotBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType, + @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere, + @Argument(ArgumentType.STRING) String reason) { + + redactBetween(start, stop, asType, ruleNumber, redactEverywhere, reason, null, false); + } + + + @ThenAction + public void redactLinesBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType, + @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere, + @Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { + + redactLinesBetween(start, stop, asType, ruleNumber, redactEverywhere, reason, legalBasis, true); + } + + + @ThenAction + public void redactNotLinesBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType, + @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere, @Argument(ArgumentType.STRING) String reason) { + redactLinesBetween(start, stop, asType, ruleNumber, redactEverywhere, reason, null, false); + } + + + @ThenAction + public void redactCell(@Argument(ArgumentType.STRING) String cellHeader, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.TYPE) String type, + @Argument(ArgumentType.BOOLEAN) boolean addAsRecommendations, @Argument(ArgumentType.STRING) String reason, + @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { + + annotateCell(cellHeader, ruleNumber, type, true, addAsRecommendations, reason, legalBasis); + } + + + @ThenAction + public void redactNotCell(@Argument(ArgumentType.STRING) String cellHeader, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.TYPE) String type, + @Argument(ArgumentType.BOOLEAN) boolean addAsRecommendations, @Argument(ArgumentType.STRING) String reason) { + + annotateCell(cellHeader, ruleNumber, type, false, addAsRecommendations, reason, null); + } + + + @ThenAction + public void redactAndRecommendByRegEx(@Argument(ArgumentType.REGEX) String pattern, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, + @Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.TYPE) String asType, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, + @Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { + + redactAndRecommendByRegEx(pattern, patternCaseInsensitive, group, asType, ruleNumber, reason, legalBasis, true); + } + + + @ThenAction + public void redactNotAndRecommendByRegEx(@Argument(ArgumentType.REGEX) String pattern, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, + @Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.TYPE) String asType, + @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason) { + + redactAndRecommendByRegEx(pattern, patternCaseInsensitive, group, asType, ruleNumber, reason, null, false); + } + + + @ThenAction + public void addRecommendationByRegEx(@Argument(ArgumentType.REGEX) String pattern, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, + @Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.TYPE) String asType) { + + Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive); + + Matcher matcher = compiledPattern.matcher(text); + + while (matcher.find()) { + String match = matcher.group(group); + if (StringUtils.isNotBlank(match) && match.length() >= 3) { + localDictionaryAdds.computeIfAbsent(RECOMMENDATION_PREFIX + asType, (x) -> new HashSet<>()).add(match); + } + } + } + + + @ThenAction + public void redactNotAndReference(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.REFERENCE_TYPE) String referenceType, + @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason) { + boolean hasRecommendationDictionary = dictionaryTypes.contains(RECOMMENDATION_PREFIX + type); - Set references = entities.stream() - .filter(entity -> entity.getType().equals(referenceType)) - .collect(Collectors.toSet()); + Set references = entities.stream().filter(entity -> entity.getType().equals(referenceType)).collect(Collectors.toSet()); entities.forEach(entity -> { - if (entity.getType().equals(type) || hasRecommendationDictionary && entity.getType() - .equals(RECOMMENDATION_PREFIX + type)) { + if (entity.getType().equals(type) || hasRecommendationDictionary && entity.getType().equals(RECOMMENDATION_PREFIX + type)) { entity.setRedaction(false); entity.setMatchedRule(ruleNumber); entity.setRedactionReason(reason); @@ -439,10 +476,38 @@ public class Section { @ThenAction - public void expandToHintAnnotationByRegEx(@Argument(ArgumentType.TYPE) String type, - @Argument(ArgumentType.STRING) String pattern, - @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, - @Argument(ArgumentType.INTEGER) int group, + public void redactIfPrecededBy(@Argument(ArgumentType.STRING) String prefix, @Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, + @Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { + + entities.forEach(entity -> { + if (entity.getType().equals(type) && searchText.indexOf(prefix + entity.getWord()) != 1) { + entity.setRedaction(true); + entity.setMatchedRule(ruleNumber); + entity.setRedactionReason(reason); + entity.setLegalBasis(legalBasis); + } + }); + } + + + @ThenAction + public void addRedaction(@Argument(ArgumentType.STRING) String value, @Argument(ArgumentType.TYPE) String asType, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, + @Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { + + Set found = findEntities(value.trim(), asType, true, true, ruleNumber, reason, legalBasis, Engine.RULE); + EntitySearchUtils.addEntitiesIgnoreRank(entities, found); + } + + + public void ignore(String type) { + + entities.removeIf(entity -> entity.getType().equals(type)); + } + + + @ThenAction + public void expandToHintAnnotationByRegEx(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.STRING) String pattern, + @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, @Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.TYPE) String asType) { Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive); @@ -470,10 +535,8 @@ public class Section { @ThenAction - public void addHintAnnotationByRegEx(@Argument(ArgumentType.REGEX) String pattern, - @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, - @Argument(ArgumentType.INTEGER) int group, - @Argument(ArgumentType.TYPE) String asType) { + public void addHintAnnotationByRegEx(@Argument(ArgumentType.REGEX) String pattern, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, + @Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.TYPE) String asType) { Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive); @@ -490,26 +553,7 @@ public class Section { @ThenAction - public void redactIfPrecededBy(@Argument(ArgumentType.STRING) String prefix, - @Argument(ArgumentType.TYPE) String type, - @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, - @Argument(ArgumentType.STRING) String reason, - @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { - - entities.forEach(entity -> { - if (entity.getType().equals(type) && searchText.indexOf(prefix + entity.getWord()) != 1) { - entity.setRedaction(true); - entity.setMatchedRule(ruleNumber); - entity.setRedactionReason(reason); - entity.setLegalBasis(legalBasis); - } - }); - } - - - @ThenAction - public void addHintAnnotation(@Argument(ArgumentType.STRING) String value, - @Argument(ArgumentType.TYPE) String asType) { + public void addHintAnnotation(@Argument(ArgumentType.STRING) String value, @Argument(ArgumentType.TYPE) String asType) { Set found = findEntities(value.trim(), asType, true, false, 0, null, null, Engine.RULE); EntitySearchUtils.addEntitiesIgnoreRank(entities, found); @@ -517,43 +561,7 @@ public class Section { @ThenAction - public void addRedaction(@Argument(ArgumentType.STRING) String value, @Argument(ArgumentType.TYPE) String asType, - @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, - @Argument(ArgumentType.STRING) String reason, - @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { - - Set found = findEntities(value.trim(), asType, true, true, ruleNumber, reason, legalBasis, Engine.RULE); - EntitySearchUtils.addEntitiesIgnoreRank(entities, found); - } - - - @ThenAction - public void redactLineAfter(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.TYPE) String asType, - @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, - @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere, - @Argument(ArgumentType.STRING) String reason, - @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { - - String[] values = StringUtils.substringsBetween(text, start, "\n"); - - if (values != null) { - for (String value : values) { - if (StringUtils.isNotBlank(value)) { - Set found = findEntities(value.trim(), asType, false, true, ruleNumber, reason, legalBasis, Engine.RULE); - EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary); - - if (redactEverywhere && !isLocal()) { - localDictionaryAdds.computeIfAbsent(asType, (x) -> new HashSet<>()).add(value.trim()); - } - } - } - } - } - - - @ThenAction - public void recommendLineAfter(@Argument(ArgumentType.STRING) String start, - @Argument(ArgumentType.TYPE) String asType) { + public void recommendLineAfter(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.TYPE) String asType) { String[] values = StringUtils.substringsBetween(text, start, "\n"); @@ -569,8 +577,7 @@ public class Section { } if (StringUtils.isNotBlank(cleanValue) && cleanValue.length() >= 3) { - localDictionaryAdds.computeIfAbsent(RECOMMENDATION_PREFIX + asType, (x) -> new HashSet<>()) - .add(cleanValue); + localDictionaryAdds.computeIfAbsent(RECOMMENDATION_PREFIX + asType, (x) -> new HashSet<>()).add(cleanValue); } } } @@ -578,163 +585,29 @@ public class Section { @ThenAction - public void redactByRegEx(@Argument(ArgumentType.REGEX) String pattern, - @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, - @Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.TYPE) String asType, - @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, - @Argument(ArgumentType.STRING) String reason, - @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { - - Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive); - - Matcher matcher = compiledPattern.matcher(searchText); - - while (matcher.find()) { - String match = matcher.group(group); - if (StringUtils.isNotBlank(match)) { - Set found = findEntities(match.trim(), asType, false, true, ruleNumber, reason, legalBasis, Engine.RULE); - EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary); - } - } - } - - - @ThenAction - public void addRecommendationByRegEx(@Argument(ArgumentType.REGEX) String pattern, - @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, - @Argument(ArgumentType.INTEGER) int group, - @Argument(ArgumentType.TYPE) String asType) { - - Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive); - - Matcher matcher = compiledPattern.matcher(text); - - while (matcher.find()) { - String match = matcher.group(group); - if (StringUtils.isNotBlank(match) && match.length() >= 3) { - localDictionaryAdds.computeIfAbsent(RECOMMENDATION_PREFIX + asType, (x) -> new HashSet<>()).add(match); - } - } - } - - - @ThenAction - public void redactAndRecommendByRegEx(@Argument(ArgumentType.REGEX) String pattern, - @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, - @Argument(ArgumentType.INTEGER) int group, - @Argument(ArgumentType.TYPE) String asType, - @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, - @Argument(ArgumentType.STRING) String reason, - @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { - - Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive); - - Matcher matcher = compiledPattern.matcher(searchText); - - while (matcher.find()) { - String match = matcher.group(group); - if (StringUtils.isNotBlank(match) && match.length() >= 3) { - localDictionaryAdds.computeIfAbsent(RECOMMENDATION_PREFIX + asType, (x) -> new HashSet<>()).add(match); - localDictionaryAdds.computeIfAbsent(asType, (x) -> new HashSet<>()).add(match); - } - } - } - - - @ThenAction - public void redactBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop, - @Argument(ArgumentType.TYPE) String asType, - @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, - @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere, - @Argument(ArgumentType.STRING) String reason, - @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { - - String[] values = StringUtils.substringsBetween(searchText, start, stop); - - if (values != null) { - for (String value : values) { - if (StringUtils.isNotBlank(value)) { - - Set found = findEntities(value.trim(), asType, false, true, ruleNumber, reason, legalBasis, Engine.RULE); - EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary); - - if (redactEverywhere && !isLocal()) { - localDictionaryAdds.computeIfAbsent(asType, (x) -> new HashSet<>()).add(value.trim()); - } - } - } - } - } - - - @ThenAction - public void redactLinesBetween(@Argument(ArgumentType.STRING) String start, - @Argument(ArgumentType.STRING) String stop, - @Argument(ArgumentType.TYPE) String asType, - @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, - @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere, - @Argument(ArgumentType.STRING) String reason, - @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { - - String[] values = StringUtils.substringsBetween(text, start, stop); - - if (values != null) { - for (String value : values) { - if (StringUtils.isNotBlank(value)) { - String[] lines = value.split("\n"); - - for (String line : lines) { - - if (line.trim().length() <= 2) { - return; - } - - Set found = findEntities(line.trim(), asType, false, true, ruleNumber, reason, legalBasis, Engine.RULE); - EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary); - - if (redactEverywhere && !isLocal()) { - localDictionaryAdds.computeIfAbsent(asType, (x) -> new HashSet<>()).add(line.trim()); - } - } - } - } - } - } - - - @ThenAction - public void highlightCell(@Argument(ArgumentType.STRING) String cellHeader, - @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, - @Argument(ArgumentType.TYPE) String type) { + public void highlightCell(@Argument(ArgumentType.STRING) String cellHeader, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.TYPE) String type) { annotateCell(cellHeader, ruleNumber, type, false, false, null, null); } - @ThenAction - public void redactCell(@Argument(ArgumentType.STRING) String cellHeader, - @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.TYPE) String type, - @Argument(ArgumentType.BOOLEAN) boolean addAsRecommendations, - @Argument(ArgumentType.STRING) String reason, - @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { + private void redactAndRecommendByRegEx(String pattern, boolean patternCaseInsensitive, int group, String asType, int ruleNumber, String reason, String legalBasis, + boolean redaction) { - annotateCell(cellHeader, ruleNumber, type, true, addAsRecommendations, reason, legalBasis); + Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive); + Matcher matcher = compiledPattern.matcher(searchText); + while (matcher.find()) { + String match = matcher.group(group); + if (StringUtils.isNotBlank(match) && match.length() >= 3) { + Set found = findEntities(match.trim(), asType, false, redaction, ruleNumber, reason, legalBasis, Engine.RULE); + EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary); + localDictionaryAdds.computeIfAbsent(RECOMMENDATION_PREFIX + asType, (x) -> new HashSet<>()).add(match); + } + } } - @ThenAction - public void redactNotCell(@Argument(ArgumentType.STRING) String cellHeader, - @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, - @Argument(ArgumentType.TYPE) String type, - @Argument(ArgumentType.BOOLEAN) boolean addAsRecommendations, - @Argument(ArgumentType.STRING) String reason) { - - annotateCell(cellHeader, ruleNumber, type, false, addAsRecommendations, reason, null); - } - - - private Set findEntities(String value, String asType, boolean caseInsensitive, boolean redacted, - int ruleNumber, String reason, String legalBasis, Engine engine) { + private Set findEntities(String value, String asType, boolean caseInsensitive, boolean redacted, int ruleNumber, String reason, String legalBasis, Engine engine) { String text = caseInsensitive ? searchText.toLowerCase() : searchText; String searchValue = caseInsensitive ? value.toLowerCase() : value; @@ -754,8 +627,35 @@ public class Section { } - private void annotateCell(String cellHeader, int ruleNumber, String type, boolean redact, - boolean addAsRecommendations, String reason, String legalBasis) { + private void redact(String type, int ruleNumber, String reason, String legalBasis, boolean redaction) { + + boolean hasRecommendationDictionary = dictionaryTypes.contains(RECOMMENDATION_PREFIX + type); + + entities.forEach(entity -> { + if (entity.getType().equals(type) || hasRecommendationDictionary && entity.getType().equals(RECOMMENDATION_PREFIX + type)) { + entity.setRedaction(redaction); + entity.setMatchedRule(ruleNumber); + entity.setRedactionReason(reason); + entity.setLegalBasis(legalBasis); + } + }); + } + + + private void redactImage(String type, int ruleNumber, String reason, String legalBasis, boolean redaction) { + + images.forEach(image -> { + if (image.getType().equals(type)) { + image.setRedaction(redaction); + image.setMatchedRule(ruleNumber); + image.setRedactionReason(reason); + image.setLegalBasis(legalBasis); + } + }); + } + + + private void annotateCell(String cellHeader, int ruleNumber, String type, boolean redact, boolean addAsRecommendations, String reason, String legalBasis) { String cleanHeaderName = cellHeader.replaceAll("\n", "").replaceAll(" ", "").replaceAll("-", ""); @@ -792,11 +692,92 @@ public class Section { while (matcher.find()) { String match = matcher.group().trim(); if (match.length() >= 3) { - localDictionaryAdds.computeIfAbsent(RECOMMENDATION_PREFIX + type, (x) -> new HashSet<>()) - .add(match); + localDictionaryAdds.computeIfAbsent(RECOMMENDATION_PREFIX + type, (x) -> new HashSet<>()).add(match); String lastname = match.split(" ")[0]; - localDictionaryAdds.computeIfAbsent(RECOMMENDATION_PREFIX + type, (x) -> new HashSet<>()) - .add(lastname); + localDictionaryAdds.computeIfAbsent(RECOMMENDATION_PREFIX + type, (x) -> new HashSet<>()).add(lastname); + } + } + } + } + } + + + private void redactLineAfter(String start, String asType, int ruleNumber, boolean redactEverywhere, String reason, String legalBasis, boolean redaction) { + + String[] values = StringUtils.substringsBetween(text, start, "\n"); + + if (values != null) { + for (String value : values) { + if (StringUtils.isNotBlank(value)) { + Set found = findEntities(value.trim(), asType, false, redaction, ruleNumber, reason, legalBasis, Engine.RULE); + EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary); + + if (redactEverywhere && !isLocal()) { + localDictionaryAdds.computeIfAbsent(asType, (x) -> new HashSet<>()).add(value.trim()); + } + } + } + } + } + + + private void redactByRegEx(String pattern, boolean patternCaseInsensitive, int group, String asType, int ruleNumber, String reason, String legalBasis, boolean redaction) { + + Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive); + + Matcher matcher = compiledPattern.matcher(searchText); + + while (matcher.find()) { + String match = matcher.group(group); + if (StringUtils.isNotBlank(match)) { + Set found = findEntities(match.trim(), asType, false, redaction, ruleNumber, reason, legalBasis, Engine.RULE); + EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary); + } + } + } + + + private void redactBetween(String start, String stop, String asType, int ruleNumber, boolean redactEverywhere, String reason, String legalBasis, boolean redaction) { + + String[] values = StringUtils.substringsBetween(searchText, start, stop); + + if (values != null) { + for (String value : values) { + if (StringUtils.isNotBlank(value)) { + + Set found = findEntities(value.trim(), asType, false, redaction, ruleNumber, reason, legalBasis, Engine.RULE); + EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary); + + if (redactEverywhere && !isLocal()) { + localDictionaryAdds.computeIfAbsent(asType, (x) -> new HashSet<>()).add(value.trim()); + } + } + } + } + } + + + private void redactLinesBetween(String start, String stop, String asType, int ruleNumber, boolean redactEverywhere, String reason, String legalBasis, boolean redaction) { + + String[] values = StringUtils.substringsBetween(text, start, stop); + + if (values != null) { + for (String value : values) { + if (StringUtils.isNotBlank(value)) { + String[] lines = value.split("\n"); + + for (String line : lines) { + + if (line.trim().length() <= 2) { + return; + } + + Set found = findEntities(line.trim(), asType, false, redaction, ruleNumber, reason, legalBasis, Engine.RULE); + EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary); + + if (redactEverywhere && !isLocal()) { + localDictionaryAdds.computeIfAbsent(asType, (x) -> new HashSet<>()).add(line.trim()); + } } } }