diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java index 10df1632..c410885d 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java @@ -18,6 +18,7 @@ import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions; import com.iqser.red.service.redaction.v1.model.ArgumentType; import com.iqser.red.service.redaction.v1.model.Engine; import com.iqser.red.service.redaction.v1.model.FileAttribute; @@ -73,12 +74,16 @@ public class Section { @Builder.Default private List sectionAreas = new ArrayList<>(); + private ManualRedactions manualRedactions; + @SuppressWarnings("unused") @WhenCondition public void addAiEntities(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.TYPE) String asType) { - Set entitiesOfType = nerEntities.stream().filter(nerEntity -> nerEntity.getType().equals(type)).collect(Collectors.toSet()); + Set entitiesOfType = nerEntities.stream() + .filter(nerEntity -> nerEntity.getType().equals(type)) + .collect(Collectors.toSet()); List values = entitiesOfType.stream().map(Entity::getWord).collect(Collectors.toList()); Set found = EntitySearchUtils.findEntities(searchText, new SearchImplementation(values, dictionary.isCaseInsensitiveDictionary(asType)), dictionary.getType(asType), new FindEntityDetails(asType, headline, sectionNumber, false, false, Engine.NER, EntityType.RECOMMENDATION)); EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary); @@ -105,15 +110,21 @@ public class Section { nerEntities.removeAll(entitiesOfType); } + @SuppressWarnings("unused") @WhenCondition - public void combineAiTypes(@Argument(ArgumentType.TYPE) String startType, @Argument(ArgumentType.TYPE) String combineTypes, - @Argument(ArgumentType.INTEGER) int maxDistanceBetween, @Argument(ArgumentType.TYPE) String asType, - @Argument(ArgumentType.INTEGER) int minPartMatches, @Argument(ArgumentType.BOOLEAN) boolean allowDuplicateTypes) { + public void combineAiTypes(@Argument(ArgumentType.TYPE) String startType, + @Argument(ArgumentType.TYPE) String combineTypes, + @Argument(ArgumentType.INTEGER) int maxDistanceBetween, + @Argument(ArgumentType.TYPE) String asType, + @Argument(ArgumentType.INTEGER) int minPartMatches, + @Argument(ArgumentType.BOOLEAN) boolean allowDuplicateTypes) { Set combineSet = Set.of(combineTypes.split(",")); - List sorted = nerEntities.stream().sorted(Comparator.comparing(Entity::getStart)).collect(Collectors.toList()); + List sorted = nerEntities.stream() + .sorted(Comparator.comparing(Entity::getStart)) + .collect(Collectors.toList()); Set found = new HashSet<>(); int start = -1; int lastEnd = -1; @@ -170,49 +181,67 @@ public class Section { } } + @SuppressWarnings("unused") @WhenCondition - public boolean fileAttributeByIdEquals(@Argument(ArgumentType.FILE_ATTRIBUTE) String id, @Argument(ArgumentType.STRING) String value) { + public boolean fileAttributeByIdEquals(@Argument(ArgumentType.FILE_ATTRIBUTE) String id, + @Argument(ArgumentType.STRING) String value) { - return fileAttributes != null && fileAttributes.stream().anyMatch(attribute -> id.equals(attribute.getId()) && value.equals(attribute.getValue())); + return fileAttributes != null && fileAttributes.stream() + .anyMatch(attribute -> id.equals(attribute.getId()) && value.equals(attribute.getValue())); } + @SuppressWarnings("unused") @WhenCondition - public boolean fileAttributeByPlaceholderEquals(@Argument(ArgumentType.FILE_ATTRIBUTE) String placeholder, @Argument(ArgumentType.STRING) String value) { + public boolean fileAttributeByPlaceholderEquals(@Argument(ArgumentType.FILE_ATTRIBUTE) String placeholder, + @Argument(ArgumentType.STRING) String value) { - return fileAttributes != null && fileAttributes.stream().anyMatch(attribute -> placeholder.equals(attribute.getPlaceholder()) && value.equals(attribute.getValue())); + return fileAttributes != null && fileAttributes.stream() + .anyMatch(attribute -> placeholder.equals(attribute.getPlaceholder()) && value.equals(attribute.getValue())); } + @SuppressWarnings("unused") @WhenCondition - public boolean fileAttributeByLabelEquals(@Argument(ArgumentType.FILE_ATTRIBUTE) String label, @Argument(ArgumentType.STRING) String value) { + public boolean fileAttributeByLabelEquals(@Argument(ArgumentType.FILE_ATTRIBUTE) String label, + @Argument(ArgumentType.STRING) String value) { - return fileAttributes != null && fileAttributes.stream().anyMatch(attribute -> label.equals(attribute.getLabel()) && value.equals(attribute.getValue())); + return fileAttributes != null && fileAttributes.stream() + .anyMatch(attribute -> label.equals(attribute.getLabel()) && value.equals(attribute.getValue())); } + @SuppressWarnings("unused") @WhenCondition - public boolean fileAttributeByIdEqualsIgnoreCase(@Argument(ArgumentType.FILE_ATTRIBUTE) String id, @Argument(ArgumentType.STRING) String value) { + public boolean fileAttributeByIdEqualsIgnoreCase(@Argument(ArgumentType.FILE_ATTRIBUTE) String id, + @Argument(ArgumentType.STRING) String value) { - return fileAttributes != null && fileAttributes.stream().anyMatch(attribute -> id.equals(attribute.getId()) && value.equalsIgnoreCase(attribute.getValue())); + return fileAttributes != null && fileAttributes.stream() + .anyMatch(attribute -> id.equals(attribute.getId()) && value.equalsIgnoreCase(attribute.getValue())); } + @SuppressWarnings("unused") @WhenCondition - public boolean fileAttributeByPlaceholderEqualsIgnoreCase(@Argument(ArgumentType.FILE_ATTRIBUTE) String placeholder, @Argument(ArgumentType.STRING) String value) { + public boolean fileAttributeByPlaceholderEqualsIgnoreCase(@Argument(ArgumentType.FILE_ATTRIBUTE) String placeholder, + @Argument(ArgumentType.STRING) String value) { return fileAttributes != null && fileAttributes.stream() .anyMatch(attribute -> placeholder.equals(attribute.getPlaceholder()) && value.equalsIgnoreCase(attribute.getValue())); } + @SuppressWarnings("unused") @WhenCondition - public boolean fileAttributeByLabelEqualsIgnoreCase(@Argument(ArgumentType.FILE_ATTRIBUTE) String label, @Argument(ArgumentType.STRING) String value) { + public boolean fileAttributeByLabelEqualsIgnoreCase(@Argument(ArgumentType.FILE_ATTRIBUTE) String label, + @Argument(ArgumentType.STRING) String value) { - return fileAttributes != null && fileAttributes.stream().anyMatch(attribute -> label.equals(attribute.getLabel()) && value.equalsIgnoreCase(attribute.getValue())); + return fileAttributes != null && fileAttributes.stream() + .anyMatch(attribute -> label.equals(attribute.getLabel()) && value.equalsIgnoreCase(attribute.getValue())); } + @SuppressWarnings("unused") @WhenCondition public boolean hasTableHeader(@Argument(ArgumentType.STRING) String headerName) { @@ -221,6 +250,7 @@ public class Section { return tabularData != null && tabularData.containsKey(cleanHeaderName); } + @SuppressWarnings("unused") @WhenCondition public boolean aiMatchesType(@Argument(ArgumentType.TYPE) String type) { @@ -228,6 +258,7 @@ public class Section { return nerEntities.stream().anyMatch(entity -> !entity.isIgnored() && entity.getType().equals(type)); } + @SuppressWarnings("unused") @WhenCondition public boolean matchesType(@Argument(ArgumentType.TYPE) String type) { @@ -235,6 +266,7 @@ public class Section { return entities.stream().anyMatch(entity -> !entity.isIgnored() && entity.getType().equals(type)); } + @SuppressWarnings("unused") @WhenCondition public boolean matchesImageType(@Argument(ArgumentType.TYPE) String type) { @@ -242,6 +274,7 @@ public class Section { return images.stream().anyMatch(image -> !image.isIgnored() && image.getType().equals(type)); } + @SuppressWarnings("unused") @WhenCondition public boolean headlineContainsWord(@Argument(ArgumentType.STRING) String word) { @@ -249,9 +282,11 @@ public class Section { return StringUtils.containsIgnoreCase(headline, word); } + @SuppressWarnings("unused") @WhenCondition - public boolean containsRegEx(@Argument(ArgumentType.STRING) String regEx, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive) { + public boolean containsRegEx(@Argument(ArgumentType.STRING) String regEx, + @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive) { var compiledPattern = Patterns.getCompiledPattern(regEx, patternCaseInsensitive); @@ -260,19 +295,26 @@ public class Section { return matcher.find(); } + @SuppressWarnings("unused") @WhenCondition - public boolean rowEquals(@Argument(ArgumentType.STRING) String headerName, @Argument(ArgumentType.STRING) String value) { + public boolean rowEquals(@Argument(ArgumentType.STRING) String headerName, + @Argument(ArgumentType.STRING) String value) { String cleanHeaderName = headerName.replaceAll("\n", "").replaceAll(" ", "").replaceAll("-", ""); - return tabularData != null && tabularData.containsKey(cleanHeaderName) && tabularData.get(cleanHeaderName).toString().equals(value); + return tabularData != null && tabularData.containsKey(cleanHeaderName) && tabularData.get(cleanHeaderName) + .toString() + .equals(value); } @ThenAction @SuppressWarnings("unused") - public void expandByPrefixRegEx(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.REGEX) String prefixPattern, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, @Argument(ArgumentType.INTEGER) int group) { + public void expandByPrefixRegEx(@Argument(ArgumentType.TYPE) String type, + @Argument(ArgumentType.REGEX) String prefixPattern, + @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, + @Argument(ArgumentType.INTEGER) int group) { expandByPrefixRegEx(type, prefixPattern, patternCaseInsensitive, group, null); } @@ -280,11 +322,15 @@ public class Section { @ThenAction @SuppressWarnings("unused") - public void expandByPrefixRegEx(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.REGEX) String prefixPattern, - @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, @Argument(ArgumentType.INTEGER) int group, + public void expandByPrefixRegEx(@Argument(ArgumentType.TYPE) String type, + @Argument(ArgumentType.REGEX) String prefixPattern, + @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, + @Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.REGEX) String valuePattern) { - if (StringUtils.isEmpty(prefixPattern)) return; + if (StringUtils.isEmpty(prefixPattern)) { + return; + } var compiledValuePattern = valuePattern == null ? null : Patterns.getCompiledPattern(valuePattern, patternCaseInsensitive); var compiledPrefixPattern = Patterns.getCompiledPattern(prefixPattern, patternCaseInsensitive); @@ -329,8 +375,10 @@ public class Section { @ThenAction @SuppressWarnings("unused") - public void expandByRegEx(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.REGEX) String suffixPattern, - @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, @Argument(ArgumentType.INTEGER) int group) { + public void expandByRegEx(@Argument(ArgumentType.TYPE) String type, + @Argument(ArgumentType.REGEX) String suffixPattern, + @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, + @Argument(ArgumentType.INTEGER) int group) { expandByRegEx(type, suffixPattern, patternCaseInsensitive, group, null); } @@ -338,11 +386,15 @@ public class Section { @ThenAction @SuppressWarnings("unused") - public void expandByRegEx(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.REGEX) String suffixPattern, - @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, @Argument(ArgumentType.INTEGER) int group, + public void expandByRegEx(@Argument(ArgumentType.TYPE) String type, + @Argument(ArgumentType.REGEX) String suffixPattern, + @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, + @Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.REGEX) String valuePattern) { - if (StringUtils.isEmpty(suffixPattern)) return; + if (StringUtils.isEmpty(suffixPattern)) { + return; + } var compiledValuePattern = valuePattern == null ? null : Patterns.getCompiledPattern(valuePattern, patternCaseInsensitive); var compiledSuffixPattern = Patterns.getCompiledPattern(suffixPattern, patternCaseInsensitive); @@ -386,7 +438,9 @@ public class Section { @ThenAction @SuppressWarnings("unused") - public void redactImage(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason, + public void redactImage(@Argument(ArgumentType.TYPE) String type, + @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, + @Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { redactImage(type, ruleNumber, reason, legalBasis, true); @@ -395,7 +449,9 @@ public class Section { @ThenAction @SuppressWarnings("unused") - public void redactNotImage(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason) { + public void redactNotImage(@Argument(ArgumentType.TYPE) String type, + @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, + @Argument(ArgumentType.STRING) String reason) { redactImage(type, ruleNumber, reason, null, false); } @@ -403,7 +459,8 @@ public class Section { @ThenAction @SuppressWarnings("unused") - public void redact(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason, + public void redact(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, + @Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { redact(type, ruleNumber, reason, legalBasis, true); @@ -412,7 +469,8 @@ public class Section { @ThenAction @SuppressWarnings("unused") - public void redactNot(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason) { + public void redactNot(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, + @Argument(ArgumentType.STRING) String reason) { redact(type, ruleNumber, reason, null, false); } @@ -420,8 +478,10 @@ public class Section { @ThenAction @SuppressWarnings("unused") - public void redactLineAfter(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.TYPE) String asType, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, - @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere, @Argument(ArgumentType.STRING) String reason, + public void redactLineAfter(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.TYPE) String asType, + @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, + @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere, + @Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { redactLineAfter(start, asType, ruleNumber, redactEverywhere, reason, legalBasis, true); @@ -430,8 +490,11 @@ public class Section { @ThenAction @SuppressWarnings("unused") - public void redactNotLineAfter(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.TYPE) String asType, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, - @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere, @Argument(ArgumentType.STRING) String reason) { + public void redactNotLineAfter(@Argument(ArgumentType.STRING) String start, + @Argument(ArgumentType.TYPE) String asType, + @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, + @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere, + @Argument(ArgumentType.STRING) String reason) { redactLineAfter(start, asType, ruleNumber, redactEverywhere, reason, null, false); @@ -440,9 +503,12 @@ public class Section { @ThenAction @SuppressWarnings("unused") - public void redactByRegEx(@Argument(ArgumentType.REGEX) String pattern, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, - @Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.TYPE) String asType, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, - @Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { + public void redactByRegEx(@Argument(ArgumentType.REGEX) String pattern, + @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, + @Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.TYPE) String asType, + @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, + @Argument(ArgumentType.STRING) String reason, + @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { redactByRegEx(pattern, patternCaseInsensitive, group, asType, ruleNumber, reason, legalBasis, true); } @@ -450,8 +516,10 @@ public class Section { @ThenAction @SuppressWarnings("unused") - public void redactNotByRegEx(@Argument(ArgumentType.REGEX) String pattern, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, - @Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.TYPE) String asType, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, + public void redactNotByRegEx(@Argument(ArgumentType.REGEX) String pattern, + @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, + @Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.TYPE) String asType, + @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason) { redactByRegEx(pattern, patternCaseInsensitive, group, asType, ruleNumber, reason, null, false); @@ -460,9 +528,12 @@ public class Section { @ThenAction @SuppressWarnings("unused") - public void redactBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType, - @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere, - @Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { + public void redactBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop, + @Argument(ArgumentType.TYPE) String asType, + @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, + @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere, + @Argument(ArgumentType.STRING) String reason, + @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { redactBetween(start, stop, asType, ruleNumber, redactEverywhere, reason, legalBasis, true); } @@ -470,8 +541,10 @@ public class Section { @ThenAction @SuppressWarnings("unused") - public void redactNotBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType, - @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere, + public void redactNotBetween(@Argument(ArgumentType.STRING) String start, + @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType, + @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, + @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere, @Argument(ArgumentType.STRING) String reason) { redactBetween(start, stop, asType, ruleNumber, redactEverywhere, reason, null, false); @@ -480,9 +553,13 @@ public class Section { @ThenAction @SuppressWarnings("unused") - public void redactLinesBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType, - @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere, - @Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { + public void redactLinesBetween(@Argument(ArgumentType.STRING) String start, + @Argument(ArgumentType.STRING) String stop, + @Argument(ArgumentType.TYPE) String asType, + @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, + @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere, + @Argument(ArgumentType.STRING) String reason, + @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { redactLinesBetween(start, stop, asType, ruleNumber, redactEverywhere, reason, legalBasis, true); } @@ -490,8 +567,11 @@ public class Section { @ThenAction @SuppressWarnings("unused") - public void redactNotLinesBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType, - @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere, + public void redactNotLinesBetween(@Argument(ArgumentType.STRING) String start, + @Argument(ArgumentType.STRING) String stop, + @Argument(ArgumentType.TYPE) String asType, + @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, + @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere, @Argument(ArgumentType.STRING) String reason) { redactLinesBetween(start, stop, asType, ruleNumber, redactEverywhere, reason, null, false); @@ -500,8 +580,10 @@ public class Section { @ThenAction @SuppressWarnings("unused") - public void redactCell(@Argument(ArgumentType.STRING) String cellHeader, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.TYPE) String type, - @Argument(ArgumentType.BOOLEAN) boolean addAsRecommendations, @Argument(ArgumentType.STRING) String reason, + public void redactCell(@Argument(ArgumentType.STRING) String cellHeader, + @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.TYPE) String type, + @Argument(ArgumentType.BOOLEAN) boolean addAsRecommendations, + @Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { annotateCell(cellHeader, ruleNumber, type, true, addAsRecommendations, reason, legalBasis); @@ -510,8 +592,11 @@ public class Section { @ThenAction @SuppressWarnings("unused") - public void redactNotCell(@Argument(ArgumentType.STRING) String cellHeader, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.TYPE) String type, - @Argument(ArgumentType.BOOLEAN) boolean addAsRecommendations, @Argument(ArgumentType.STRING) String reason) { + public void redactNotCell(@Argument(ArgumentType.STRING) String cellHeader, + @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, + @Argument(ArgumentType.TYPE) String type, + @Argument(ArgumentType.BOOLEAN) boolean addAsRecommendations, + @Argument(ArgumentType.STRING) String reason) { annotateCell(cellHeader, ruleNumber, type, false, addAsRecommendations, reason, null); } @@ -519,9 +604,13 @@ public class Section { @ThenAction @SuppressWarnings("unused") - public void redactAndRecommendByRegEx(@Argument(ArgumentType.REGEX) String pattern, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, - @Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.TYPE) String asType, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, - @Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { + public void redactAndRecommendByRegEx(@Argument(ArgumentType.REGEX) String pattern, + @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, + @Argument(ArgumentType.INTEGER) int group, + @Argument(ArgumentType.TYPE) String asType, + @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, + @Argument(ArgumentType.STRING) String reason, + @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { redactAndRecommendByRegEx(pattern, patternCaseInsensitive, group, asType, ruleNumber, reason, legalBasis, true); } @@ -529,9 +618,12 @@ public class Section { @ThenAction @SuppressWarnings("unused") - public void redactNotAndRecommendByRegEx(@Argument(ArgumentType.REGEX) String pattern, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, - @Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.TYPE) String asType, - @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason) { + public void redactNotAndRecommendByRegEx(@Argument(ArgumentType.REGEX) String pattern, + @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, + @Argument(ArgumentType.INTEGER) int group, + @Argument(ArgumentType.TYPE) String asType, + @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, + @Argument(ArgumentType.STRING) String reason) { redactAndRecommendByRegEx(pattern, patternCaseInsensitive, group, asType, ruleNumber, reason, null, false); } @@ -539,8 +631,10 @@ public class Section { @ThenAction @SuppressWarnings("unused") - public void addRecommendationByRegEx(@Argument(ArgumentType.REGEX) String pattern, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, - @Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.TYPE) String asType) { + public void addRecommendationByRegEx(@Argument(ArgumentType.REGEX) String pattern, + @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, + @Argument(ArgumentType.INTEGER) int group, + @Argument(ArgumentType.TYPE) String asType) { Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive); @@ -557,11 +651,14 @@ public class Section { @ThenAction @SuppressWarnings("unused") - public void redactNotAndReference(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.REFERENCE_TYPE) String referenceType, - @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason) { + public void redactNotAndReference(@Argument(ArgumentType.TYPE) String type, + @Argument(ArgumentType.REFERENCE_TYPE) String referenceType, + @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, + @Argument(ArgumentType.STRING) String reason) { - - Set references = entities.stream().filter(entity -> entity.getType().equals(referenceType)).collect(Collectors.toSet()); + Set references = entities.stream() + .filter(entity -> entity.getType().equals(referenceType)) + .collect(Collectors.toSet()); entities.forEach(entity -> { if (entity.getType().equals(type)) { @@ -576,8 +673,11 @@ public class Section { @ThenAction @SuppressWarnings("unused") - public void redactIfPrecededBy(@Argument(ArgumentType.STRING) String prefix, @Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, - @Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { + public void redactIfPrecededBy(@Argument(ArgumentType.STRING) String prefix, + @Argument(ArgumentType.TYPE) String type, + @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, + @Argument(ArgumentType.STRING) String reason, + @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { entities.forEach(entity -> { if (entity.getType().equals(type) && searchText.indexOf(prefix + entity.getWord()) != 1) { @@ -592,13 +692,16 @@ public class Section { @ThenAction @SuppressWarnings("unused") - public void addRedaction(@Argument(ArgumentType.STRING) String value, @Argument(ArgumentType.TYPE) String asType, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, - @Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { + public void addRedaction(@Argument(ArgumentType.STRING) String value, @Argument(ArgumentType.TYPE) String asType, + @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, + @Argument(ArgumentType.STRING) String reason, + @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { Set found = findEntities(value.trim(), asType, true, true, ruleNumber, reason, legalBasis, Engine.RULE, false); EntitySearchUtils.addEntitiesIgnoreRank(entities, found); } + @ThenAction @SuppressWarnings("unused") public void ignore(@Argument(ArgumentType.TYPE) String type) { @@ -606,18 +709,22 @@ public class Section { entities.removeIf(entity -> entity.getType().equals(type) && entity.getEntityType().equals(EntityType.ENTITY)); } + @ThenAction @SuppressWarnings("unused") public void ignoreRecommendations(@Argument(ArgumentType.TYPE) String type) { - entities.removeIf(entity -> entity.getType().equals(type) && entity.getEntityType().equals(EntityType.RECOMMENDATION)); + entities.removeIf(entity -> entity.getType().equals(type) && entity.getEntityType() + .equals(EntityType.RECOMMENDATION)); } @ThenAction @SuppressWarnings("unused") - public void expandToFalsePositiveByRegEx(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.STRING) String pattern, - @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, @Argument(ArgumentType.INTEGER) int group) { + public void expandToFalsePositiveByRegEx(@Argument(ArgumentType.TYPE) String type, + @Argument(ArgumentType.STRING) String pattern, + @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, + @Argument(ArgumentType.INTEGER) int group) { Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive); @@ -646,8 +753,10 @@ public class Section { @ThenAction @SuppressWarnings("unused") - public void addHintAnnotationByRegEx(@Argument(ArgumentType.REGEX) String pattern, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, - @Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.TYPE) String asType) { + public void addHintAnnotationByRegEx(@Argument(ArgumentType.REGEX) String pattern, + @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, + @Argument(ArgumentType.INTEGER) int group, + @Argument(ArgumentType.TYPE) String asType) { Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive); @@ -665,7 +774,8 @@ public class Section { @ThenAction @SuppressWarnings("unused") - public void addHintAnnotation(@Argument(ArgumentType.STRING) String value, @Argument(ArgumentType.TYPE) String asType) { + public void addHintAnnotation(@Argument(ArgumentType.STRING) String value, + @Argument(ArgumentType.TYPE) String asType) { Set found = findEntities(value.trim(), asType, true, false, 0, null, null, Engine.RULE, false); EntitySearchUtils.addEntitiesIgnoreRank(entities, found); @@ -674,7 +784,8 @@ public class Section { @ThenAction @SuppressWarnings("unused") - public void recommendLineAfter(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.TYPE) String asType) { + public void recommendLineAfter(@Argument(ArgumentType.STRING) String start, + @Argument(ArgumentType.TYPE) String asType) { String[] values = StringUtils.substringsBetween(text, start, "\n"); @@ -699,7 +810,9 @@ public class Section { @ThenAction @SuppressWarnings("unused") - public void highlightCell(@Argument(ArgumentType.STRING) String cellHeader, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.TYPE) String type) { + public void highlightCell(@Argument(ArgumentType.STRING) String cellHeader, + @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, + @Argument(ArgumentType.TYPE) String type) { annotateCell(cellHeader, ruleNumber, type, false, false, null, null); } @@ -707,7 +820,9 @@ public class Section { @ThenAction @SuppressWarnings("unused") - public void redactSection(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason, + public void redactSection(@Argument(ArgumentType.TYPE) String type, + @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, + @Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { for (SectionArea sectionArea : sectionAreas) { @@ -740,8 +855,8 @@ public class Section { } - private void redactAndRecommendByRegEx(String pattern, boolean patternCaseInsensitive, int group, String asType, int ruleNumber, String reason, String legalBasis, - boolean redaction) { + private void redactAndRecommendByRegEx(String pattern, boolean patternCaseInsensitive, int group, String asType, + int ruleNumber, String reason, String legalBasis, boolean redaction) { Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive); Matcher matcher = compiledPattern.matcher(searchText); @@ -756,11 +871,12 @@ public class Section { } - private Set findEntities(String value, String asType, boolean caseInsensitive, boolean redacted, int ruleNumber, String reason, String legalBasis, Engine engine, boolean asRecommendation) { + private Set findEntities(String value, String asType, boolean caseInsensitive, boolean redacted, + int ruleNumber, String reason, String legalBasis, Engine engine, + boolean asRecommendation) { String text = caseInsensitive ? searchText.toLowerCase() : searchText; - Set found = EntitySearchUtils.findEntities(text, new SearchImplementation(value, caseInsensitive), dictionary.getType(asType), - new FindEntityDetails(asType, headline, sectionNumber, false, false, engine, asRecommendation ? EntityType.RECOMMENDATION : EntityType.ENTITY)); + Set found = EntitySearchUtils.findEntities(text, new SearchImplementation(value, caseInsensitive), dictionary.getType(asType), new FindEntityDetails(asType, headline, sectionNumber, false, false, engine, asRecommendation ? EntityType.RECOMMENDATION : EntityType.ENTITY)); found.forEach(entity -> { if (redacted) { entity.setRedaction(true); @@ -770,7 +886,26 @@ public class Section { } }); - return EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary); + return applyResizeRedactions(EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary)); + } + + + private Set applyResizeRedactions(Set entitiesWithPositions) { + + if (manualRedactions == null || manualRedactions.getResizeRedactions() == null || manualRedactions.getResizeRedactions().isEmpty()){ + return entitiesWithPositions; + } + + entitiesWithPositions.forEach(e -> e.getPositionSequences().forEach(pos -> { + manualRedactions.getResizeRedactions().forEach(resize -> { + if (resize.getAnnotationId().equals(pos.getId()) && resize.getValue().length() < e.getWord().length() && e.getWord().contains(resize.getValue())) { + int start = e.getWord().indexOf(resize.getValue()); + e.setStart(e.getStart() + start); + e.setEnd(e.getEnd() - resize.getValue().length()); + } + }); + })); + return entitiesWithPositions; } @@ -800,7 +935,8 @@ public class Section { } - private void annotateCell(String cellHeader, int ruleNumber, String type, boolean redact, boolean addAsRecommendations, String reason, String legalBasis) { + private void annotateCell(String cellHeader, int ruleNumber, String type, boolean redact, + boolean addAsRecommendations, String reason, String legalBasis) { String cleanHeaderName = cellHeader.replaceAll("\n", "").replaceAll(" ", "").replaceAll("-", ""); @@ -847,7 +983,8 @@ public class Section { } - private void redactLineAfter(String start, String asType, int ruleNumber, boolean redactEverywhere, String reason, String legalBasis, boolean redaction) { + private void redactLineAfter(String start, String asType, int ruleNumber, boolean redactEverywhere, String reason, + String legalBasis, boolean redaction) { String[] values = StringUtils.substringsBetween(text, start, "\n"); @@ -866,7 +1003,8 @@ public class Section { } - private void redactByRegEx(String pattern, boolean patternCaseInsensitive, int group, String asType, int ruleNumber, String reason, String legalBasis, boolean redaction) { + private void redactByRegEx(String pattern, boolean patternCaseInsensitive, int group, String asType, int ruleNumber, + String reason, String legalBasis, boolean redaction) { Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive); @@ -882,7 +1020,8 @@ public class Section { } - private void redactBetween(String start, String stop, String asType, int ruleNumber, boolean redactEverywhere, String reason, String legalBasis, boolean redaction) { + private void redactBetween(String start, String stop, String asType, int ruleNumber, boolean redactEverywhere, + String reason, String legalBasis, boolean redaction) { String[] values = StringUtils.substringsBetween(searchText, start, stop); @@ -902,7 +1041,8 @@ public class Section { } - private void redactLinesBetween(String start, String stop, String asType, int ruleNumber, boolean redactEverywhere, String reason, String legalBasis, boolean redaction) { + private void redactLinesBetween(String start, String stop, String asType, int ruleNumber, boolean redactEverywhere, + String reason, String legalBasis, boolean redaction) { String[] values = StringUtils.substringsBetween(text, start, stop); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java index eeaec117..592192b0 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java @@ -14,6 +14,7 @@ import org.kie.api.runtime.KieContainer; import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions; import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval; import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization; import com.iqser.red.service.redaction.v1.model.AnalyzeRequest; @@ -146,6 +147,7 @@ public class EntityRedactionService { .images(reanalysisSection.getImages()) .sectionAreas(reanalysisSection.getSectionAreas()) .fileAttributes(analyzeRequest.getFileAttributes()) + .manualRedactions(analyzeRequest.getManualRedactions()) .build(), reanalysisSection.getSearchableText())); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index 2e74a990..a67ce82e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -30,7 +30,9 @@ import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizati import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; import com.iqser.red.storage.commons.StorageAutoConfiguration; import com.iqser.red.storage.commons.service.StorageService; + import lombok.SneakyThrows; + import org.apache.commons.io.IOUtils; import org.junit.After; import org.junit.Before; @@ -216,16 +218,16 @@ public class RedactionIntegrationTest { } - private void mockDictionaryCalls(Long version){ + private void mockDictionaryCalls(Long version) { when(dictionaryClient.getDictionaryForType(VERTEBRATE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(VERTEBRATE, false)); when(dictionaryClient.getDictionaryForType(ADDRESS + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(ADDRESS, false)); when(dictionaryClient.getDictionaryForType(AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(AUTHOR, false)); - when(dictionaryClient.getDictionaryForType(SPONSOR + ":" + TEST_DOSSIER_TEMPLATE_ID,version)).thenReturn(getDictionaryResponse(SPONSOR, false)); + when(dictionaryClient.getDictionaryForType(SPONSOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(SPONSOR, false)); when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(NO_REDACTION_INDICATOR, false)); when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(REDACTION_INDICATOR, false)); when(dictionaryClient.getDictionaryForType(HINT_ONLY + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(HINT_ONLY, false)); - when(dictionaryClient.getDictionaryForType(MUST_REDACT + ":" + TEST_DOSSIER_TEMPLATE_ID,version)).thenReturn(getDictionaryResponse(MUST_REDACT, false)); + when(dictionaryClient.getDictionaryForType(MUST_REDACT + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(MUST_REDACT, false)); when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(PUBLISHED_INFORMATION, false)); when(dictionaryClient.getDictionaryForType(TEST_METHOD + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(TEST_METHOD, false)); when(dictionaryClient.getDictionaryForType(PII + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(PII, false)); @@ -240,6 +242,7 @@ public class RedactionIntegrationTest { } + @Test public void test270Rotated() { @@ -493,14 +496,12 @@ public class RedactionIntegrationTest { deleted.add("David Chubb"); deleted.add("mouse"); - reanlysisVersions.put("mouse", 3L); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(3L); when(dictionaryClient.getDictionaryForType(VERTEBRATE, null)).thenReturn(getDictionaryResponse(VERTEBRATE, false)); - start = System.currentTimeMillis(); ManualRedactions manualRedactions = new ManualRedactions(); @@ -671,7 +672,6 @@ public class RedactionIntegrationTest { when(dictionaryClient.getDictionaryForType(VERTEBRATE, null)).thenReturn(getDictionaryResponse(VERTEBRATE, false)); - start = System.currentTimeMillis(); ManualRedactions manualRedactions = new ManualRedactions(); @@ -726,49 +726,48 @@ public class RedactionIntegrationTest { var colors = objectMapper.readValue(colorsResource.getInputStream(), Colors.class); ClassPathResource typeResource = new ClassPathResource("colors/types.json"); - TypeReference> typeRefForTypes = new TypeReference<>(){}; + TypeReference> typeRefForTypes = new TypeReference<>() { + }; List types = objectMapper.readValue(typeResource.getInputStream(), typeRefForTypes); - AnalyzeRequest request = prepareStorage("files/new/PublishedInformationTest.pdf"); analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); ManualRedactions manualRedactions = new ManualRedactions(); - manualRedactions.getIdsToRemove().add(IdRemoval.builder() - .annotationId("308dab9015bfafd911568cffe0a7f7de") - .fileId(TEST_FILE_ID) - .status(AnnotationStatus.APPROVED) - .requestDate(OffsetDateTime.of(2022,05,23,8,30,07,475479, ZoneOffset.UTC)) - .processedDate(OffsetDateTime.of(2022,05,23,8,30,07,483651, ZoneOffset.UTC)) - .build()); - - manualRedactions.getForceRedactions().add(ManualForceRedaction.builder() - .annotationId("0b56ea1a87c83f351df177315af94f0d") - .fileId(TEST_FILE_ID) - .status(AnnotationStatus.APPROVED) - .requestDate(OffsetDateTime.of(2022,05,23,9,30,15,4653, ZoneOffset.UTC)) - .processedDate(OffsetDateTime.of(2022,05,23,9,30,15,794, ZoneOffset.UTC)) - .build()); - - manualRedactions.getIdsToRemove().add(IdRemoval.builder() - .annotationId("0b56ea1a87c83f351df177315af94f0d") - .fileId(TEST_FILE_ID) - .status(AnnotationStatus.APPROVED) - .requestDate(OffsetDateTime.of(2022,05,23,8,30,23,961721, ZoneOffset.UTC)) - .processedDate(OffsetDateTime.of(2022,05,23,8,30,23,96528, ZoneOffset.UTC)) - .build()); + manualRedactions.getIdsToRemove() + .add(IdRemoval.builder() + .annotationId("308dab9015bfafd911568cffe0a7f7de") + .fileId(TEST_FILE_ID) + .status(AnnotationStatus.APPROVED) + .requestDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 07, 475479, ZoneOffset.UTC)) + .processedDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 07, 483651, ZoneOffset.UTC)) + .build()); + manualRedactions.getForceRedactions() + .add(ManualForceRedaction.builder() + .annotationId("0b56ea1a87c83f351df177315af94f0d") + .fileId(TEST_FILE_ID) + .status(AnnotationStatus.APPROVED) + .requestDate(OffsetDateTime.of(2022, 05, 23, 9, 30, 15, 4653, ZoneOffset.UTC)) + .processedDate(OffsetDateTime.of(2022, 05, 23, 9, 30, 15, 794, ZoneOffset.UTC)) + .build()); + manualRedactions.getIdsToRemove() + .add(IdRemoval.builder() + .annotationId("0b56ea1a87c83f351df177315af94f0d") + .fileId(TEST_FILE_ID) + .status(AnnotationStatus.APPROVED) + .requestDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 23, 961721, ZoneOffset.UTC)) + .processedDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 23, 96528, ZoneOffset.UTC)) + .build()); request.setManualRedactions(manualRedactions); - AnalyzeResult result = analyzeService.analyze(request); - AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder() - .manualRedactions(manualRedactions) - .colors(colors) - .types(types) + .manualRedactions(manualRedactions) + .colors(colors) + .types(types) .dossierId(TEST_DOSSIER_ID) .fileId(TEST_FILE_ID) .build()); @@ -808,6 +807,61 @@ public class RedactionIntegrationTest { } + @Test + public void testFindDictionaryEntryInResizedEntryPosition() throws IOException { + + System.out.println("testResizeDictFound"); + + ClassPathResource colorsResource = new ClassPathResource("colors/colors.json"); + var colors = objectMapper.readValue(colorsResource.getInputStream(), Colors.class); + + ClassPathResource typeResource = new ClassPathResource("colors/types.json"); + TypeReference> typeRefForTypes = new TypeReference<>() { + }; + List types = objectMapper.readValue(typeResource.getInputStream(), typeRefForTypes); + + long start = System.currentTimeMillis(); + + AnalyzeRequest request = prepareStorage("files/new/ResizeRedactionTestDoc.pdf"); + + ManualRedactions manualRedactions = new ManualRedactions(); + + manualRedactions.getResizeRedactions() + .add(ManualResizeRedaction.builder() + .annotationId("fd5ee82900beb7383f00480dea417569") + .fileId(TEST_FILE_ID) + .status(AnnotationStatus.APPROVED) + .requestDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 07, 475479, ZoneOffset.UTC)) + .processedDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 07, 483651, ZoneOffset.UTC)) + .value("Regina") + .positions(List.of(new Rectangle(147.86f, 225.9895f, 31.17696f, 10.048125f, 1))) + .textBefore("Switzerland Contact point: ") + .textAfter(" Dorn Phone:") + .build()); + + request.setManualRedactions(manualRedactions); + + analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); + AnalyzeResult result = analyzeService.analyze(request); + + AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder() + .manualRedactions(manualRedactions) + .colors(colors) + .types(types) + .dossierId(TEST_DOSSIER_ID) + .fileId(TEST_FILE_ID) + .build()); + + try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated.pdf")) { + fileOutputStream.write(annotateResponse.getDocument()); + } + long end = System.currentTimeMillis(); + + System.out.println("duration: " + (end - start)); + System.out.println("numberOfPages: " + result.getNumberOfPages()); + } + + @Test public void testManualRedaction() throws IOException { @@ -1607,15 +1661,16 @@ public class RedactionIntegrationTest { } } + @Test public void testExpandByPrefixRegEx() throws IOException { assertThat(dictionary.get(AUTHOR).contains("Robinson")); - assertThat(! dictionary.get(AUTHOR).contains("Mrs. Robinson")); + assertThat(!dictionary.get(AUTHOR).contains("Mrs. Robinson")); assertThat(dictionary.get(AUTHOR).contains("Bojangles")); - assertThat(! dictionary.get(AUTHOR).contains("Mr. Bojangles")); + assertThat(!dictionary.get(AUTHOR).contains("Mr. Bojangles")); assertThat(dictionary.get(AUTHOR).contains("Tambourine Man")); - assertThat(! dictionary.get(AUTHOR).contains("Mr. Tambourine Man")); + assertThat(!dictionary.get(AUTHOR).contains("Mr. Tambourine Man")); String fileName = "files/mr-mrs.pdf"; String outputFileName = OsUtils.getTemporaryDirectory() + "/Annotated.pdf"; @@ -1646,6 +1701,7 @@ public class RedactionIntegrationTest { assertThat(values).contains("Mr. Tambourine Man"); } + @SneakyThrows private AnalyzeRequest prepareStorage(InputStream stream) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/PII.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/PII.txt index f3fd7713..141cd1fe 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/PII.txt +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/PII.txt @@ -9,3 +9,4 @@ Naka-27 Aomachi, Nomi, Ishikawa 923-1101, Japan, JP Özgür U. Reyhan Sude Halide Nurullah Xinyi Y. Tao +Dorn diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/new/ResizeRedactionTestDoc.pdf b/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/new/ResizeRedactionTestDoc.pdf new file mode 100644 index 00000000..9ceebf0d Binary files /dev/null and b/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/new/ResizeRedactionTestDoc.pdf differ