diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java index 17a5fad6..491e9ad6 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java @@ -8,11 +8,9 @@ import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; @@ -74,9 +72,7 @@ public class Section { public void addAiEntities(String type, String asType) { - Set entitiesOfType = nerEntities.stream() - .filter(nerEntity -> nerEntity.getType().equals(type)) - .collect(Collectors.toSet()); + Set entitiesOfType = nerEntities.stream().filter(nerEntity -> nerEntity.getType().equals(type)).collect(Collectors.toSet()); Set values = entitiesOfType.stream().map(Entity::getWord).collect(Collectors.toSet()); Set found = EntitySearchUtils.find(searchText, values, asType, headline, sectionNumber, false, false, Engine.NER, true); EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary); @@ -84,13 +80,12 @@ public class Section { Set finalResult = new HashSet<>(); // Only keep Entities with correct offsets from AI Service. - Iterator itty = found.iterator(); - while (itty.hasNext()) { - Entity current = itty.next(); + for (Entity current : found) { boolean foundSameOffsets = false; for (Entity entity : entitiesOfType) { if (entity.getStart().equals(current.getStart()) && entity.getEnd().equals(current.getEnd())) { foundSameOffsets = true; + break; } } if (foundSameOffsets) { @@ -105,14 +100,11 @@ public class Section { } - public void combineAiTypes(String startType, String combineTypes, int maxDistanceBetween, String asType, - int minPartMatches, boolean allowDuplicateTypes) { + public void combineAiTypes(String startType, String combineTypes, int maxDistanceBetween, String asType, int minPartMatches, boolean allowDuplicateTypes) { Set combineSet = Set.of(combineTypes.split(",")); - List sorted = nerEntities.stream() - .sorted(Comparator.comparing(Entity::getStart)) - .collect(Collectors.toList()); + List sorted = nerEntities.stream().sorted(Comparator.comparing(Entity::getStart)).collect(Collectors.toList()); Set found = new HashSet<>(); int start = -1; int lastEnd = -1; @@ -170,81 +162,66 @@ public class Section { } - @WhenCondition - public boolean fileAttributeByIdEquals(@Argument(ArgumentType.FILE_ATTRIBUTE) String id, - @Argument(ArgumentType.STRING) String value) { + private Set findEntities(String value, String asType, boolean caseInsensitive, boolean redacted, int ruleNumber, String reason, String legalBasis, Engine engine) { - return fileAttributes != null && fileAttributes.stream() - .filter(attribute -> id.equals(attribute.getId()) && value.equals(attribute.getValue())) - .findFirst() - .isPresent(); + String text = caseInsensitive ? searchText.toLowerCase() : searchText; + String searchValue = caseInsensitive ? value.toLowerCase() : value; + + Set found = EntitySearchUtils.find(text, Set.of(searchValue), asType, headline, sectionNumber, false, false, engine, false); + + found.forEach(entity -> { + if (redacted) { + entity.setRedaction(true); + entity.setMatchedRule(ruleNumber); + entity.setRedactionReason(reason); + entity.setLegalBasis(legalBasis); + } + }); + + return EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary); } @WhenCondition - public boolean fileAttributeByPlaceholderEquals(@Argument(ArgumentType.FILE_ATTRIBUTE) String placeholder, - @Argument(ArgumentType.STRING) String value) { + public boolean fileAttributeByIdEquals(@Argument(ArgumentType.FILE_ATTRIBUTE) String id, @Argument(ArgumentType.STRING) String value) { - return fileAttributes != null && fileAttributes.stream() - .filter(attribute -> placeholder.equals(attribute.getPlaceholder()) && value.equals(attribute.getValue())) - .findFirst() - .isPresent(); + return fileAttributes != null && fileAttributes.stream().anyMatch(attribute -> id.equals(attribute.getId()) && value.equals(attribute.getValue())); } @WhenCondition - public boolean fileAttributeByLabelEquals(@Argument(ArgumentType.FILE_ATTRIBUTE) String label, - @Argument(ArgumentType.STRING) String value) { + public boolean fileAttributeByPlaceholderEquals(@Argument(ArgumentType.FILE_ATTRIBUTE) String placeholder, @Argument(ArgumentType.STRING) String value) { - return fileAttributes != null && fileAttributes.stream() - .filter(attribute -> label.equals(attribute.getLabel()) && value.equals(attribute.getValue())) - .findFirst() - .isPresent(); + return fileAttributes != null && fileAttributes.stream().anyMatch(attribute -> placeholder.equals(attribute.getPlaceholder()) && value.equals(attribute.getValue())); } @WhenCondition - public boolean fileAttributeByIdEqualsIgnoreCase(@Argument(ArgumentType.FILE_ATTRIBUTE) String id, - @Argument(ArgumentType.STRING) String value) { + public boolean fileAttributeByLabelEquals(@Argument(ArgumentType.FILE_ATTRIBUTE) String label, @Argument(ArgumentType.STRING) String value) { - return fileAttributes != null && fileAttributes.stream() - .filter(attribute -> id.equals(attribute.getId()) && value.equalsIgnoreCase(attribute.getValue())) - .findFirst() - .isPresent(); + return fileAttributes != null && fileAttributes.stream().anyMatch(attribute -> label.equals(attribute.getLabel()) && value.equals(attribute.getValue())); } @WhenCondition - public boolean fileAttributeByPlaceholderEqualsIgnoreCase(@Argument(ArgumentType.FILE_ATTRIBUTE) String placeholder, - @Argument(ArgumentType.STRING) String value) { + public boolean fileAttributeByIdEqualsIgnoreCase(@Argument(ArgumentType.FILE_ATTRIBUTE) String id, @Argument(ArgumentType.STRING) String value) { - return fileAttributes != null && fileAttributes.stream() - .filter(attribute -> placeholder.equals(attribute.getPlaceholder()) && value.equalsIgnoreCase(attribute.getValue())) - .findFirst() - .isPresent(); + return fileAttributes != null && fileAttributes.stream().anyMatch(attribute -> id.equals(attribute.getId()) && value.equalsIgnoreCase(attribute.getValue())); } @WhenCondition - public boolean fileAttributeByLabelEqualsIgnoreCase(@Argument(ArgumentType.FILE_ATTRIBUTE) String label, - @Argument(ArgumentType.STRING) String value) { + public boolean fileAttributeByPlaceholderEqualsIgnoreCase(@Argument(ArgumentType.FILE_ATTRIBUTE) String placeholder, @Argument(ArgumentType.STRING) String value) { return fileAttributes != null && fileAttributes.stream() - .filter(attribute -> label.equals(attribute.getLabel()) && value.equalsIgnoreCase(attribute.getValue())) - .findFirst() - .isPresent(); + .anyMatch(attribute -> placeholder.equals(attribute.getPlaceholder()) && value.equalsIgnoreCase(attribute.getValue())); } @WhenCondition - public boolean rowEquals(@Argument(ArgumentType.STRING) String headerName, - @Argument(ArgumentType.STRING) String value) { + public boolean fileAttributeByLabelEqualsIgnoreCase(@Argument(ArgumentType.FILE_ATTRIBUTE) String label, @Argument(ArgumentType.STRING) String value) { - String cleanHeaderName = headerName.replaceAll("\n", "").replaceAll(" ", "").replaceAll("-", ""); - - return tabularData != null && tabularData.containsKey(cleanHeaderName) && tabularData.get(cleanHeaderName) - .toString() - .equals(value); + return fileAttributes != null && fileAttributes.stream().anyMatch(attribute -> label.equals(attribute.getLabel()) && value.equalsIgnoreCase(attribute.getValue())); } @@ -284,21 +261,26 @@ public class Section { } + @WhenCondition + public boolean rowEquals(@Argument(ArgumentType.STRING) String headerName, @Argument(ArgumentType.STRING) String value) { + + String cleanHeaderName = headerName.replaceAll("\n", "").replaceAll(" ", "").replaceAll("-", ""); + + return tabularData != null && tabularData.containsKey(cleanHeaderName) && tabularData.get(cleanHeaderName).toString().equals(value); + } + + @ThenAction - public void expandByRegEx(@Argument(ArgumentType.TYPE) String type, - @Argument(ArgumentType.REGEX) String suffixPattern, - @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, - @Argument(ArgumentType.INTEGER) int group) { + public void expandByRegEx(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.REGEX) String suffixPattern, + @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, @Argument(ArgumentType.INTEGER) int group) { expandByRegEx(type, suffixPattern, patternCaseInsensitive, group, null); } @ThenAction - public void expandByRegEx(@Argument(ArgumentType.TYPE) String type, - @Argument(ArgumentType.REGEX) String suffixPattern, - @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, - @Argument(ArgumentType.INTEGER) int group, + public void expandByRegEx(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.REGEX) String suffixPattern, + @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, @Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.REGEX) String valuePattern) { Pattern compiledValuePattern = null; @@ -341,14 +323,18 @@ public class Section { @ThenAction - public void redactImage(@Argument(ArgumentType.TYPE) String type, - @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, - @Argument(ArgumentType.STRING) String reason, + public void redactImage(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { + redactImage(type, ruleNumber, reason, legalBasis, true); + } + + + private void redactImage(String type, int ruleNumber, String reason, String legalBasis, boolean redaction) { + images.forEach(image -> { if (image.getType().equals(type)) { - image.setRedaction(true); + image.setRedaction(redaction); image.setMatchedRule(ruleNumber); image.setRedactionReason(reason); image.setLegalBasis(legalBasis); @@ -358,16 +344,27 @@ public class Section { @ThenAction - public void redact(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, - @Argument(ArgumentType.STRING) String reason, + public void redactNotImage(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason) { + + redactImage(type, ruleNumber, reason, null, false); + } + + + @ThenAction + public void redact(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { + redact(type, ruleNumber, reason, legalBasis, true); + } + + + private void redact(String type, int ruleNumber, String reason, String legalBasis, boolean redaction) { + boolean hasRecommendationDictionary = dictionaryTypes.contains(RECOMMENDATION_PREFIX + type); entities.forEach(entity -> { - if (entity.getType().equals(type) || hasRecommendationDictionary && entity.getType() - .equals(RECOMMENDATION_PREFIX + type)) { - entity.setRedaction(true); + if (entity.getType().equals(type) || hasRecommendationDictionary && entity.getType().equals(RECOMMENDATION_PREFIX + type)) { + entity.setRedaction(redaction); entity.setMatchedRule(ruleNumber); entity.setRedactionReason(reason); entity.setLegalBasis(legalBasis); @@ -377,169 +374,29 @@ public class Section { @ThenAction - public void redactNotImage(@Argument(ArgumentType.TYPE) String type, - @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, - @Argument(ArgumentType.STRING) String reason) { + public void redactNot(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason) { - images.forEach(image -> { - if (image.getType().equals(type)) { - image.setRedaction(false); - image.setMatchedRule(ruleNumber); - image.setRedactionReason(reason); - } - }); + redact(type, ruleNumber, reason, null, false); } @ThenAction - public void redactNot(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, - @Argument(ArgumentType.STRING) String reason) { - - boolean hasRecommendationDictionary = dictionaryTypes.contains(RECOMMENDATION_PREFIX + type); - - entities.forEach(entity -> { - if (entity.getType().equals(type) || hasRecommendationDictionary && entity.getType() - .equals(RECOMMENDATION_PREFIX + type)) { - entity.setRedaction(false); - entity.setMatchedRule(ruleNumber); - entity.setRedactionReason(reason); - } - }); - } - - - public void ignore(String type) { - - entities.removeIf(entity -> entity.getType().equals(type)); - } - - - @ThenAction - public void redactNotAndReference(@Argument(ArgumentType.TYPE) String type, - @Argument(ArgumentType.REFERENCE_TYPE) String referenceType, - @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, - @Argument(ArgumentType.STRING) String reason) { - - boolean hasRecommendationDictionary = dictionaryTypes.contains(RECOMMENDATION_PREFIX + type); - - Set references = entities.stream() - .filter(entity -> entity.getType().equals(referenceType)) - .collect(Collectors.toSet()); - - entities.forEach(entity -> { - if (entity.getType().equals(type) || hasRecommendationDictionary && entity.getType() - .equals(RECOMMENDATION_PREFIX + type)) { - entity.setRedaction(false); - entity.setMatchedRule(ruleNumber); - entity.setRedactionReason(reason); - entity.setReferences(references); - } - }); - } - - - @ThenAction - public void expandToHintAnnotationByRegEx(@Argument(ArgumentType.TYPE) String type, - @Argument(ArgumentType.STRING) String pattern, - @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, - @Argument(ArgumentType.INTEGER) int group, - @Argument(ArgumentType.TYPE) String asType) { - - Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive); - - Set expanded = new HashSet<>(); - for (Entity entity : entities) { - - if (!entity.getType().equals(type) || entity.getTextAfter() == null) { - continue; - } - - Matcher matcher = compiledPattern.matcher(entity.getTextAfter()); - - while (matcher.find()) { - String match = matcher.group(group); - if (StringUtils.isNotBlank(match)) { - expanded.addAll(findEntities(entity.getWord() + match, asType, false, false, 0, null, null, Engine.RULE)); - } - } - } - - EntitySearchUtils.addEntitiesWithHigherRank(entities, expanded, dictionary); - EntitySearchUtils.removeEntitiesContainedInLarger(entities); - } - - - @ThenAction - public void addHintAnnotationByRegEx(@Argument(ArgumentType.REGEX) String pattern, - @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, - @Argument(ArgumentType.INTEGER) int group, - @Argument(ArgumentType.TYPE) String asType) { - - Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive); - - Matcher matcher = compiledPattern.matcher(searchText); - - while (matcher.find()) { - String match = matcher.group(group); - if (StringUtils.isNotBlank(match)) { - Set found = findEntities(match.trim(), asType, false, false, 0, null, null, Engine.RULE); - EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary); - } - } - } - - - @ThenAction - public void redactIfPrecededBy(@Argument(ArgumentType.STRING) String prefix, - @Argument(ArgumentType.TYPE) String type, - @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, - @Argument(ArgumentType.STRING) String reason, - @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { - - entities.forEach(entity -> { - if (entity.getType().equals(type) && searchText.indexOf(prefix + entity.getWord()) != 1) { - entity.setRedaction(true); - entity.setMatchedRule(ruleNumber); - entity.setRedactionReason(reason); - entity.setLegalBasis(legalBasis); - } - }); - } - - - @ThenAction - public void addHintAnnotation(@Argument(ArgumentType.STRING) String value, - @Argument(ArgumentType.TYPE) String asType) { - - Set found = findEntities(value.trim(), asType, true, false, 0, null, null, Engine.RULE); - EntitySearchUtils.addEntitiesIgnoreRank(entities, found); - } - - - @ThenAction - public void addRedaction(@Argument(ArgumentType.STRING) String value, @Argument(ArgumentType.TYPE) String asType, - @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, - @Argument(ArgumentType.STRING) String reason, - @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { - - Set found = findEntities(value.trim(), asType, true, true, ruleNumber, reason, legalBasis, Engine.RULE); - EntitySearchUtils.addEntitiesIgnoreRank(entities, found); - } - - - @ThenAction - public void redactLineAfter(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.TYPE) String asType, - @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, - @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere, - @Argument(ArgumentType.STRING) String reason, + public void redactLineAfter(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.TYPE) String asType, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, + @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere, @Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { + redactLineAfter(start, asType, ruleNumber, redactEverywhere, reason, legalBasis, true); + } + + + private void redactLineAfter(String start, String asType, int ruleNumber, boolean redactEverywhere, String reason, String legalBasis, boolean redaction) { + String[] values = StringUtils.substringsBetween(text, start, "\n"); if (values != null) { for (String value : values) { if (StringUtils.isNotBlank(value)) { - Set found = findEntities(value.trim(), asType, false, true, ruleNumber, reason, legalBasis, Engine.RULE); + Set found = findEntities(value.trim(), asType, false, redaction, ruleNumber, reason, legalBasis, Engine.RULE); EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary); if (redactEverywhere && !isLocal()) { @@ -552,38 +409,24 @@ public class Section { @ThenAction - public void recommendLineAfter(@Argument(ArgumentType.STRING) String start, - @Argument(ArgumentType.TYPE) String asType) { + public void redactNotLineAfter(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.TYPE) String asType, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, + @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere, @Argument(ArgumentType.STRING) String reason) { - String[] values = StringUtils.substringsBetween(text, start, "\n"); + redactLineAfter(start, asType, ruleNumber, redactEverywhere, reason, null, false); - if (values != null) { - for (String value : values) { - - String trimmedValue = value.trim(); - String cleanValue; - if (trimmedValue.startsWith(":")) { - cleanValue = trimmedValue.substring(1).trim(); - } else { - cleanValue = trimmedValue; - } - - if (StringUtils.isNotBlank(cleanValue) && cleanValue.length() >= 3) { - localDictionaryAdds.computeIfAbsent(RECOMMENDATION_PREFIX + asType, (x) -> new HashSet<>()) - .add(cleanValue); - } - } - } } @ThenAction - public void redactByRegEx(@Argument(ArgumentType.REGEX) String pattern, - @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, - @Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.TYPE) String asType, - @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, - @Argument(ArgumentType.STRING) String reason, - @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { + public void redactByRegEx(@Argument(ArgumentType.REGEX) String pattern, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, + @Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.TYPE) String asType, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, + @Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { + + redactByRegEx(pattern, patternCaseInsensitive, group, asType, ruleNumber, reason, legalBasis, true); + } + + + private void redactByRegEx(String pattern, boolean patternCaseInsensitive, int group, String asType, int ruleNumber, String reason, String legalBasis, boolean redaction) { Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive); @@ -592,7 +435,7 @@ public class Section { while (matcher.find()) { String match = matcher.group(group); if (StringUtils.isNotBlank(match)) { - Set found = findEntities(match.trim(), asType, false, true, ruleNumber, reason, legalBasis, Engine.RULE); + Set found = findEntities(match.trim(), asType, false, redaction, ruleNumber, reason, legalBasis, Engine.RULE); EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary); } } @@ -600,54 +443,24 @@ public class Section { @ThenAction - public void addRecommendationByRegEx(@Argument(ArgumentType.REGEX) String pattern, - @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, - @Argument(ArgumentType.INTEGER) int group, - @Argument(ArgumentType.TYPE) String asType) { + public void redactNotByRegEx(@Argument(ArgumentType.REGEX) String pattern, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, + @Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.TYPE) String asType, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, + @Argument(ArgumentType.STRING) String reason) { - Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive); - - Matcher matcher = compiledPattern.matcher(text); - - while (matcher.find()) { - String match = matcher.group(group); - if (StringUtils.isNotBlank(match) && match.length() >= 3) { - localDictionaryAdds.computeIfAbsent(RECOMMENDATION_PREFIX + asType, (x) -> new HashSet<>()).add(match); - } - } + redactByRegEx(pattern, patternCaseInsensitive, group, asType, ruleNumber, reason, null, false); } @ThenAction - public void redactAndRecommendByRegEx(@Argument(ArgumentType.REGEX) String pattern, - @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, - @Argument(ArgumentType.INTEGER) int group, - @Argument(ArgumentType.TYPE) String asType, - @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, - @Argument(ArgumentType.STRING) String reason, - @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { + public void redactBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType, + @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere, + @Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { - Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive); - - Matcher matcher = compiledPattern.matcher(searchText); - - while (matcher.find()) { - String match = matcher.group(group); - if (StringUtils.isNotBlank(match) && match.length() >= 3) { - localDictionaryAdds.computeIfAbsent(RECOMMENDATION_PREFIX + asType, (x) -> new HashSet<>()).add(match); - localDictionaryAdds.computeIfAbsent(asType, (x) -> new HashSet<>()).add(match); - } - } + redactBetween(start, stop, asType, ruleNumber, redactEverywhere, reason, legalBasis, true); } - @ThenAction - public void redactBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop, - @Argument(ArgumentType.TYPE) String asType, - @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, - @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere, - @Argument(ArgumentType.STRING) String reason, - @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { + private void redactBetween(String start, String stop, String asType, int ruleNumber, boolean redactEverywhere, String reason, String legalBasis, boolean redaction) { String[] values = StringUtils.substringsBetween(searchText, start, stop); @@ -655,7 +468,7 @@ public class Section { for (String value : values) { if (StringUtils.isNotBlank(value)) { - Set found = findEntities(value.trim(), asType, false, true, ruleNumber, reason, legalBasis, Engine.RULE); + Set found = findEntities(value.trim(), asType, false, redaction, ruleNumber, reason, legalBasis, Engine.RULE); EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary); if (redactEverywhere && !isLocal()) { @@ -668,13 +481,24 @@ public class Section { @ThenAction - public void redactLinesBetween(@Argument(ArgumentType.STRING) String start, - @Argument(ArgumentType.STRING) String stop, - @Argument(ArgumentType.TYPE) String asType, - @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, - @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere, - @Argument(ArgumentType.STRING) String reason, - @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { + public void redactNotBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType, + @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere, + @Argument(ArgumentType.STRING) String reason) { + + redactBetween(start, stop, asType, ruleNumber, redactEverywhere, reason, null, false); + } + + + @ThenAction + public void redactLinesBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType, + @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere, + @Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { + + redactLinesBetween(start, stop, asType, ruleNumber, redactEverywhere, reason, legalBasis, true); + } + + + private void redactLinesBetween(String start, String stop, String asType, int ruleNumber, boolean redactEverywhere, String reason, String legalBasis, boolean redaction) { String[] values = StringUtils.substringsBetween(text, start, stop); @@ -689,7 +513,7 @@ public class Section { return; } - Set found = findEntities(line.trim(), asType, false, true, ruleNumber, reason, legalBasis, Engine.RULE); + Set found = findEntities(line.trim(), asType, false, redaction, ruleNumber, reason, legalBasis, Engine.RULE); EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary); if (redactEverywhere && !isLocal()) { @@ -703,59 +527,24 @@ public class Section { @ThenAction - public void highlightCell(@Argument(ArgumentType.STRING) String cellHeader, - @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, - @Argument(ArgumentType.TYPE) String type) { + public void redactNotLinesBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType, + @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere, + @Argument(ArgumentType.STRING) String reason) { - annotateCell(cellHeader, ruleNumber, type, false, false, null, null); + redactLinesBetween(start, stop, asType, ruleNumber, redactEverywhere, reason, null, false); } @ThenAction - public void redactCell(@Argument(ArgumentType.STRING) String cellHeader, - @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.TYPE) String type, - @Argument(ArgumentType.BOOLEAN) boolean addAsRecommendations, - @Argument(ArgumentType.STRING) String reason, + public void redactCell(@Argument(ArgumentType.STRING) String cellHeader, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.TYPE) String type, + @Argument(ArgumentType.BOOLEAN) boolean addAsRecommendations, @Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { annotateCell(cellHeader, ruleNumber, type, true, addAsRecommendations, reason, legalBasis); } - @ThenAction - public void redactNotCell(@Argument(ArgumentType.STRING) String cellHeader, - @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, - @Argument(ArgumentType.TYPE) String type, - @Argument(ArgumentType.BOOLEAN) boolean addAsRecommendations, - @Argument(ArgumentType.STRING) String reason) { - - annotateCell(cellHeader, ruleNumber, type, false, addAsRecommendations, reason, null); - } - - - private Set findEntities(String value, String asType, boolean caseInsensitive, boolean redacted, - int ruleNumber, String reason, String legalBasis, Engine engine) { - - String text = caseInsensitive ? searchText.toLowerCase() : searchText; - String searchValue = caseInsensitive ? value.toLowerCase() : value; - - Set found = EntitySearchUtils.find(text, Set.of(searchValue), asType, headline, sectionNumber, false, false, engine, false); - - found.forEach(entity -> { - if (redacted) { - entity.setRedaction(true); - entity.setMatchedRule(ruleNumber); - entity.setRedactionReason(reason); - entity.setLegalBasis(legalBasis); - } - }); - - return EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary); - } - - - private void annotateCell(String cellHeader, int ruleNumber, String type, boolean redact, - boolean addAsRecommendations, String reason, String legalBasis) { + private void annotateCell(String cellHeader, int ruleNumber, String type, boolean redact, boolean addAsRecommendations, String reason, String legalBasis) { String cleanHeaderName = cellHeader.replaceAll("\n", "").replaceAll(" ", "").replaceAll("-", ""); @@ -792,11 +581,9 @@ public class Section { while (matcher.find()) { String match = matcher.group().trim(); if (match.length() >= 3) { - localDictionaryAdds.computeIfAbsent(RECOMMENDATION_PREFIX + type, (x) -> new HashSet<>()) - .add(match); + localDictionaryAdds.computeIfAbsent(RECOMMENDATION_PREFIX + type, (x) -> new HashSet<>()).add(match); String lastname = match.split(" ")[0]; - localDictionaryAdds.computeIfAbsent(RECOMMENDATION_PREFIX + type, (x) -> new HashSet<>()) - .add(lastname); + localDictionaryAdds.computeIfAbsent(RECOMMENDATION_PREFIX + type, (x) -> new HashSet<>()).add(lastname); } } } @@ -804,6 +591,200 @@ public class Section { } + @ThenAction + public void redactNotCell(@Argument(ArgumentType.STRING) String cellHeader, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.TYPE) String type, + @Argument(ArgumentType.BOOLEAN) boolean addAsRecommendations, @Argument(ArgumentType.STRING) String reason) { + + annotateCell(cellHeader, ruleNumber, type, false, addAsRecommendations, reason, null); + } + + + @ThenAction + public void redactAndRecommendByRegEx(@Argument(ArgumentType.REGEX) String pattern, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, + @Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.TYPE) String asType, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, + @Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { + + redactAndRecommendByRegEx(pattern, patternCaseInsensitive, group, asType, ruleNumber, reason, legalBasis, true); + } + + + private void redactAndRecommendByRegEx(String pattern, boolean patternCaseInsensitive, int group, String asType, int ruleNumber, String reason, String legalBasis, + boolean redaction) { + + Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive); + Matcher matcher = compiledPattern.matcher(searchText); + while (matcher.find()) { + String match = matcher.group(group); + if (StringUtils.isNotBlank(match) && match.length() >= 3) { + Set found = findEntities(match.trim(), asType, false, redaction, ruleNumber, reason, legalBasis, Engine.RULE); + EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary); + localDictionaryAdds.computeIfAbsent(RECOMMENDATION_PREFIX + asType, (x) -> new HashSet<>()).add(match); + } + } + } + + + @ThenAction + public void addRecommendationByRegEx(@Argument(ArgumentType.REGEX) String pattern, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, + @Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.TYPE) String asType) { + + Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive); + + Matcher matcher = compiledPattern.matcher(text); + + while (matcher.find()) { + String match = matcher.group(group); + if (StringUtils.isNotBlank(match) && match.length() >= 3) { + localDictionaryAdds.computeIfAbsent(RECOMMENDATION_PREFIX + asType, (x) -> new HashSet<>()).add(match); + } + } + } + + + @ThenAction + public void redactNotAndRecommendByRegEx(@Argument(ArgumentType.REGEX) String pattern, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, + @Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.TYPE) String asType, + @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason) { + + redactAndRecommendByRegEx(pattern, patternCaseInsensitive, group, asType, ruleNumber, reason, null, false); + } + + + @ThenAction + public void redactNotAndReference(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.REFERENCE_TYPE) String referenceType, + @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason) { + + boolean hasRecommendationDictionary = dictionaryTypes.contains(RECOMMENDATION_PREFIX + type); + + Set references = entities.stream().filter(entity -> entity.getType().equals(referenceType)).collect(Collectors.toSet()); + + entities.forEach(entity -> { + if (entity.getType().equals(type) || hasRecommendationDictionary && entity.getType().equals(RECOMMENDATION_PREFIX + type)) { + entity.setRedaction(false); + entity.setMatchedRule(ruleNumber); + entity.setRedactionReason(reason); + entity.setReferences(references); + } + }); + } + + + @ThenAction + public void redactIfPrecededBy(@Argument(ArgumentType.STRING) String prefix, @Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, + @Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { + + entities.forEach(entity -> { + if (entity.getType().equals(type) && searchText.indexOf(prefix + entity.getWord()) != 1) { + entity.setRedaction(true); + entity.setMatchedRule(ruleNumber); + entity.setRedactionReason(reason); + entity.setLegalBasis(legalBasis); + } + }); + } + + + @ThenAction + public void addRedaction(@Argument(ArgumentType.STRING) String value, @Argument(ArgumentType.TYPE) String asType, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, + @Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { + + Set found = findEntities(value.trim(), asType, true, true, ruleNumber, reason, legalBasis, Engine.RULE); + EntitySearchUtils.addEntitiesIgnoreRank(entities, found); + } + + + public void ignore(String type) { + + entities.removeIf(entity -> entity.getType().equals(type)); + } + + + @ThenAction + public void expandToHintAnnotationByRegEx(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.STRING) String pattern, + @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, @Argument(ArgumentType.INTEGER) int group, + @Argument(ArgumentType.TYPE) String asType) { + + Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive); + + Set expanded = new HashSet<>(); + for (Entity entity : entities) { + + if (!entity.getType().equals(type) || entity.getTextAfter() == null) { + continue; + } + + Matcher matcher = compiledPattern.matcher(entity.getTextAfter()); + + while (matcher.find()) { + String match = matcher.group(group); + if (StringUtils.isNotBlank(match)) { + expanded.addAll(findEntities(entity.getWord() + match, asType, false, false, 0, null, null, Engine.RULE)); + } + } + } + + EntitySearchUtils.addEntitiesWithHigherRank(entities, expanded, dictionary); + EntitySearchUtils.removeEntitiesContainedInLarger(entities); + } + + + @ThenAction + public void addHintAnnotationByRegEx(@Argument(ArgumentType.REGEX) String pattern, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, + @Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.TYPE) String asType) { + + Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive); + + Matcher matcher = compiledPattern.matcher(searchText); + + while (matcher.find()) { + String match = matcher.group(group); + if (StringUtils.isNotBlank(match)) { + Set found = findEntities(match.trim(), asType, false, false, 0, null, null, Engine.RULE); + EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary); + } + } + } + + + @ThenAction + public void addHintAnnotation(@Argument(ArgumentType.STRING) String value, @Argument(ArgumentType.TYPE) String asType) { + + Set found = findEntities(value.trim(), asType, true, false, 0, null, null, Engine.RULE); + EntitySearchUtils.addEntitiesIgnoreRank(entities, found); + } + + + @ThenAction + public void recommendLineAfter(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.TYPE) String asType) { + + String[] values = StringUtils.substringsBetween(text, start, "\n"); + + if (values != null) { + for (String value : values) { + + String trimmedValue = value.trim(); + String cleanValue; + if (trimmedValue.startsWith(":")) { + cleanValue = trimmedValue.substring(1).trim(); + } else { + cleanValue = trimmedValue; + } + + if (StringUtils.isNotBlank(cleanValue) && cleanValue.length() >= 3) { + localDictionaryAdds.computeIfAbsent(RECOMMENDATION_PREFIX + asType, (x) -> new HashSet<>()).add(cleanValue); + } + } + } + } + + + @ThenAction + public void highlightCell(@Argument(ArgumentType.STRING) String cellHeader, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.TYPE) String type) { + + annotateCell(cellHeader, ruleNumber, type, false, false, null, null); + } + + @Retention(RetentionPolicy.RUNTIME) @Target(ElementType.METHOD) public @interface WhenCondition {