RED-4064: Fixed find dictionary entries at position where rules entries are resized to smaller entries
This commit is contained in:
deiflaender
parent
50b2908763
commit
778c975649
@ -18,6 +18,7 @@ import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
|
||||
import com.iqser.red.service.redaction.v1.model.ArgumentType;
|
||||
import com.iqser.red.service.redaction.v1.model.Engine;
|
||||
import com.iqser.red.service.redaction.v1.model.FileAttribute;
|
||||
@ -73,12 +74,16 @@ public class Section {
|
||||
@Builder.Default
|
||||
private List<SectionArea> sectionAreas = new ArrayList<>();
|
||||
|
||||
private ManualRedactions manualRedactions;
|
||||
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
@WhenCondition
|
||||
public void addAiEntities(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.TYPE) String asType) {
|
||||
|
||||
Set<Entity> entitiesOfType = nerEntities.stream().filter(nerEntity -> nerEntity.getType().equals(type)).collect(Collectors.toSet());
|
||||
Set<Entity> entitiesOfType = nerEntities.stream()
|
||||
.filter(nerEntity -> nerEntity.getType().equals(type))
|
||||
.collect(Collectors.toSet());
|
||||
List<String> values = entitiesOfType.stream().map(Entity::getWord).collect(Collectors.toList());
|
||||
Set<Entity> found = EntitySearchUtils.findEntities(searchText, new SearchImplementation(values, dictionary.isCaseInsensitiveDictionary(asType)), dictionary.getType(asType), new FindEntityDetails(asType, headline, sectionNumber, false, false, Engine.NER, EntityType.RECOMMENDATION));
|
||||
EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary);
|
||||
@ -105,15 +110,21 @@ public class Section {
|
||||
nerEntities.removeAll(entitiesOfType);
|
||||
}
|
||||
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
@WhenCondition
|
||||
public void combineAiTypes(@Argument(ArgumentType.TYPE) String startType, @Argument(ArgumentType.TYPE) String combineTypes,
|
||||
@Argument(ArgumentType.INTEGER) int maxDistanceBetween, @Argument(ArgumentType.TYPE) String asType,
|
||||
@Argument(ArgumentType.INTEGER) int minPartMatches, @Argument(ArgumentType.BOOLEAN) boolean allowDuplicateTypes) {
|
||||
public void combineAiTypes(@Argument(ArgumentType.TYPE) String startType,
|
||||
@Argument(ArgumentType.TYPE) String combineTypes,
|
||||
@Argument(ArgumentType.INTEGER) int maxDistanceBetween,
|
||||
@Argument(ArgumentType.TYPE) String asType,
|
||||
@Argument(ArgumentType.INTEGER) int minPartMatches,
|
||||
@Argument(ArgumentType.BOOLEAN) boolean allowDuplicateTypes) {
|
||||
|
||||
Set<String> combineSet = Set.of(combineTypes.split(","));
|
||||
|
||||
List<Entity> sorted = nerEntities.stream().sorted(Comparator.comparing(Entity::getStart)).collect(Collectors.toList());
|
||||
List<Entity> sorted = nerEntities.stream()
|
||||
.sorted(Comparator.comparing(Entity::getStart))
|
||||
.collect(Collectors.toList());
|
||||
Set<Entity> found = new HashSet<>();
|
||||
int start = -1;
|
||||
int lastEnd = -1;
|
||||
@ -170,49 +181,67 @@ public class Section {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
@WhenCondition
|
||||
public boolean fileAttributeByIdEquals(@Argument(ArgumentType.FILE_ATTRIBUTE) String id, @Argument(ArgumentType.STRING) String value) {
|
||||
public boolean fileAttributeByIdEquals(@Argument(ArgumentType.FILE_ATTRIBUTE) String id,
|
||||
@Argument(ArgumentType.STRING) String value) {
|
||||
|
||||
return fileAttributes != null && fileAttributes.stream().anyMatch(attribute -> id.equals(attribute.getId()) && value.equals(attribute.getValue()));
|
||||
return fileAttributes != null && fileAttributes.stream()
|
||||
.anyMatch(attribute -> id.equals(attribute.getId()) && value.equals(attribute.getValue()));
|
||||
}
|
||||
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
@WhenCondition
|
||||
public boolean fileAttributeByPlaceholderEquals(@Argument(ArgumentType.FILE_ATTRIBUTE) String placeholder, @Argument(ArgumentType.STRING) String value) {
|
||||
public boolean fileAttributeByPlaceholderEquals(@Argument(ArgumentType.FILE_ATTRIBUTE) String placeholder,
|
||||
@Argument(ArgumentType.STRING) String value) {
|
||||
|
||||
return fileAttributes != null && fileAttributes.stream().anyMatch(attribute -> placeholder.equals(attribute.getPlaceholder()) && value.equals(attribute.getValue()));
|
||||
return fileAttributes != null && fileAttributes.stream()
|
||||
.anyMatch(attribute -> placeholder.equals(attribute.getPlaceholder()) && value.equals(attribute.getValue()));
|
||||
}
|
||||
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
@WhenCondition
|
||||
public boolean fileAttributeByLabelEquals(@Argument(ArgumentType.FILE_ATTRIBUTE) String label, @Argument(ArgumentType.STRING) String value) {
|
||||
public boolean fileAttributeByLabelEquals(@Argument(ArgumentType.FILE_ATTRIBUTE) String label,
|
||||
@Argument(ArgumentType.STRING) String value) {
|
||||
|
||||
return fileAttributes != null && fileAttributes.stream().anyMatch(attribute -> label.equals(attribute.getLabel()) && value.equals(attribute.getValue()));
|
||||
return fileAttributes != null && fileAttributes.stream()
|
||||
.anyMatch(attribute -> label.equals(attribute.getLabel()) && value.equals(attribute.getValue()));
|
||||
}
|
||||
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
@WhenCondition
|
||||
public boolean fileAttributeByIdEqualsIgnoreCase(@Argument(ArgumentType.FILE_ATTRIBUTE) String id, @Argument(ArgumentType.STRING) String value) {
|
||||
public boolean fileAttributeByIdEqualsIgnoreCase(@Argument(ArgumentType.FILE_ATTRIBUTE) String id,
|
||||
@Argument(ArgumentType.STRING) String value) {
|
||||
|
||||
return fileAttributes != null && fileAttributes.stream().anyMatch(attribute -> id.equals(attribute.getId()) && value.equalsIgnoreCase(attribute.getValue()));
|
||||
return fileAttributes != null && fileAttributes.stream()
|
||||
.anyMatch(attribute -> id.equals(attribute.getId()) && value.equalsIgnoreCase(attribute.getValue()));
|
||||
}
|
||||
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
@WhenCondition
|
||||
public boolean fileAttributeByPlaceholderEqualsIgnoreCase(@Argument(ArgumentType.FILE_ATTRIBUTE) String placeholder, @Argument(ArgumentType.STRING) String value) {
|
||||
public boolean fileAttributeByPlaceholderEqualsIgnoreCase(@Argument(ArgumentType.FILE_ATTRIBUTE) String placeholder,
|
||||
@Argument(ArgumentType.STRING) String value) {
|
||||
|
||||
return fileAttributes != null && fileAttributes.stream()
|
||||
.anyMatch(attribute -> placeholder.equals(attribute.getPlaceholder()) && value.equalsIgnoreCase(attribute.getValue()));
|
||||
}
|
||||
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
@WhenCondition
|
||||
public boolean fileAttributeByLabelEqualsIgnoreCase(@Argument(ArgumentType.FILE_ATTRIBUTE) String label, @Argument(ArgumentType.STRING) String value) {
|
||||
public boolean fileAttributeByLabelEqualsIgnoreCase(@Argument(ArgumentType.FILE_ATTRIBUTE) String label,
|
||||
@Argument(ArgumentType.STRING) String value) {
|
||||
|
||||
return fileAttributes != null && fileAttributes.stream().anyMatch(attribute -> label.equals(attribute.getLabel()) && value.equalsIgnoreCase(attribute.getValue()));
|
||||
return fileAttributes != null && fileAttributes.stream()
|
||||
.anyMatch(attribute -> label.equals(attribute.getLabel()) && value.equalsIgnoreCase(attribute.getValue()));
|
||||
}
|
||||
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
@WhenCondition
|
||||
public boolean hasTableHeader(@Argument(ArgumentType.STRING) String headerName) {
|
||||
@ -221,6 +250,7 @@ public class Section {
|
||||
return tabularData != null && tabularData.containsKey(cleanHeaderName);
|
||||
}
|
||||
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
@WhenCondition
|
||||
public boolean aiMatchesType(@Argument(ArgumentType.TYPE) String type) {
|
||||
@ -228,6 +258,7 @@ public class Section {
|
||||
return nerEntities.stream().anyMatch(entity -> !entity.isIgnored() && entity.getType().equals(type));
|
||||
}
|
||||
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
@WhenCondition
|
||||
public boolean matchesType(@Argument(ArgumentType.TYPE) String type) {
|
||||
@ -235,6 +266,7 @@ public class Section {
|
||||
return entities.stream().anyMatch(entity -> !entity.isIgnored() && entity.getType().equals(type));
|
||||
}
|
||||
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
@WhenCondition
|
||||
public boolean matchesImageType(@Argument(ArgumentType.TYPE) String type) {
|
||||
@ -242,6 +274,7 @@ public class Section {
|
||||
return images.stream().anyMatch(image -> !image.isIgnored() && image.getType().equals(type));
|
||||
}
|
||||
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
@WhenCondition
|
||||
public boolean headlineContainsWord(@Argument(ArgumentType.STRING) String word) {
|
||||
@ -249,9 +282,11 @@ public class Section {
|
||||
return StringUtils.containsIgnoreCase(headline, word);
|
||||
}
|
||||
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
@WhenCondition
|
||||
public boolean containsRegEx(@Argument(ArgumentType.STRING) String regEx, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive) {
|
||||
public boolean containsRegEx(@Argument(ArgumentType.STRING) String regEx,
|
||||
@Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive) {
|
||||
|
||||
var compiledPattern = Patterns.getCompiledPattern(regEx, patternCaseInsensitive);
|
||||
|
||||
@ -260,19 +295,26 @@ public class Section {
|
||||
return matcher.find();
|
||||
}
|
||||
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
@WhenCondition
|
||||
public boolean rowEquals(@Argument(ArgumentType.STRING) String headerName, @Argument(ArgumentType.STRING) String value) {
|
||||
public boolean rowEquals(@Argument(ArgumentType.STRING) String headerName,
|
||||
@Argument(ArgumentType.STRING) String value) {
|
||||
|
||||
String cleanHeaderName = headerName.replaceAll("\n", "").replaceAll(" ", "").replaceAll("-", "");
|
||||
|
||||
return tabularData != null && tabularData.containsKey(cleanHeaderName) && tabularData.get(cleanHeaderName).toString().equals(value);
|
||||
return tabularData != null && tabularData.containsKey(cleanHeaderName) && tabularData.get(cleanHeaderName)
|
||||
.toString()
|
||||
.equals(value);
|
||||
}
|
||||
|
||||
|
||||
@ThenAction
|
||||
@SuppressWarnings("unused")
|
||||
public void expandByPrefixRegEx(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.REGEX) String prefixPattern, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, @Argument(ArgumentType.INTEGER) int group) {
|
||||
public void expandByPrefixRegEx(@Argument(ArgumentType.TYPE) String type,
|
||||
@Argument(ArgumentType.REGEX) String prefixPattern,
|
||||
@Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
|
||||
@Argument(ArgumentType.INTEGER) int group) {
|
||||
|
||||
expandByPrefixRegEx(type, prefixPattern, patternCaseInsensitive, group, null);
|
||||
}
|
||||
@ -280,11 +322,15 @@ public class Section {
|
||||
|
||||
@ThenAction
|
||||
@SuppressWarnings("unused")
|
||||
public void expandByPrefixRegEx(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.REGEX) String prefixPattern,
|
||||
@Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, @Argument(ArgumentType.INTEGER) int group,
|
||||
public void expandByPrefixRegEx(@Argument(ArgumentType.TYPE) String type,
|
||||
@Argument(ArgumentType.REGEX) String prefixPattern,
|
||||
@Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
|
||||
@Argument(ArgumentType.INTEGER) int group,
|
||||
@Argument(ArgumentType.REGEX) String valuePattern) {
|
||||
|
||||
if (StringUtils.isEmpty(prefixPattern)) return;
|
||||
if (StringUtils.isEmpty(prefixPattern)) {
|
||||
return;
|
||||
}
|
||||
|
||||
var compiledValuePattern = valuePattern == null ? null : Patterns.getCompiledPattern(valuePattern, patternCaseInsensitive);
|
||||
var compiledPrefixPattern = Patterns.getCompiledPattern(prefixPattern, patternCaseInsensitive);
|
||||
@ -329,8 +375,10 @@ public class Section {
|
||||
|
||||
@ThenAction
|
||||
@SuppressWarnings("unused")
|
||||
public void expandByRegEx(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.REGEX) String suffixPattern,
|
||||
@Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, @Argument(ArgumentType.INTEGER) int group) {
|
||||
public void expandByRegEx(@Argument(ArgumentType.TYPE) String type,
|
||||
@Argument(ArgumentType.REGEX) String suffixPattern,
|
||||
@Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
|
||||
@Argument(ArgumentType.INTEGER) int group) {
|
||||
|
||||
expandByRegEx(type, suffixPattern, patternCaseInsensitive, group, null);
|
||||
}
|
||||
@ -338,11 +386,15 @@ public class Section {
|
||||
|
||||
@ThenAction
|
||||
@SuppressWarnings("unused")
|
||||
public void expandByRegEx(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.REGEX) String suffixPattern,
|
||||
@Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, @Argument(ArgumentType.INTEGER) int group,
|
||||
public void expandByRegEx(@Argument(ArgumentType.TYPE) String type,
|
||||
@Argument(ArgumentType.REGEX) String suffixPattern,
|
||||
@Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
|
||||
@Argument(ArgumentType.INTEGER) int group,
|
||||
@Argument(ArgumentType.REGEX) String valuePattern) {
|
||||
|
||||
if (StringUtils.isEmpty(suffixPattern)) return;
|
||||
if (StringUtils.isEmpty(suffixPattern)) {
|
||||
return;
|
||||
}
|
||||
|
||||
var compiledValuePattern = valuePattern == null ? null : Patterns.getCompiledPattern(valuePattern, patternCaseInsensitive);
|
||||
var compiledSuffixPattern = Patterns.getCompiledPattern(suffixPattern, patternCaseInsensitive);
|
||||
@ -386,7 +438,9 @@ public class Section {
|
||||
|
||||
@ThenAction
|
||||
@SuppressWarnings("unused")
|
||||
public void redactImage(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason,
|
||||
public void redactImage(@Argument(ArgumentType.TYPE) String type,
|
||||
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
|
||||
@Argument(ArgumentType.STRING) String reason,
|
||||
@Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
|
||||
|
||||
redactImage(type, ruleNumber, reason, legalBasis, true);
|
||||
@ -395,7 +449,9 @@ public class Section {
|
||||
|
||||
@ThenAction
|
||||
@SuppressWarnings("unused")
|
||||
public void redactNotImage(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason) {
|
||||
public void redactNotImage(@Argument(ArgumentType.TYPE) String type,
|
||||
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
|
||||
@Argument(ArgumentType.STRING) String reason) {
|
||||
|
||||
redactImage(type, ruleNumber, reason, null, false);
|
||||
}
|
||||
@ -403,7 +459,8 @@ public class Section {
|
||||
|
||||
@ThenAction
|
||||
@SuppressWarnings("unused")
|
||||
public void redact(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason,
|
||||
public void redact(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
|
||||
@Argument(ArgumentType.STRING) String reason,
|
||||
@Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
|
||||
|
||||
redact(type, ruleNumber, reason, legalBasis, true);
|
||||
@ -412,7 +469,8 @@ public class Section {
|
||||
|
||||
@ThenAction
|
||||
@SuppressWarnings("unused")
|
||||
public void redactNot(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason) {
|
||||
public void redactNot(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
|
||||
@Argument(ArgumentType.STRING) String reason) {
|
||||
|
||||
redact(type, ruleNumber, reason, null, false);
|
||||
}
|
||||
@ -420,8 +478,10 @@ public class Section {
|
||||
|
||||
@ThenAction
|
||||
@SuppressWarnings("unused")
|
||||
public void redactLineAfter(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.TYPE) String asType, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
|
||||
@Argument(ArgumentType.BOOLEAN) boolean redactEverywhere, @Argument(ArgumentType.STRING) String reason,
|
||||
public void redactLineAfter(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.TYPE) String asType,
|
||||
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
|
||||
@Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
|
||||
@Argument(ArgumentType.STRING) String reason,
|
||||
@Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
|
||||
|
||||
redactLineAfter(start, asType, ruleNumber, redactEverywhere, reason, legalBasis, true);
|
||||
@ -430,8 +490,11 @@ public class Section {
|
||||
|
||||
@ThenAction
|
||||
@SuppressWarnings("unused")
|
||||
public void redactNotLineAfter(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.TYPE) String asType, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
|
||||
@Argument(ArgumentType.BOOLEAN) boolean redactEverywhere, @Argument(ArgumentType.STRING) String reason) {
|
||||
public void redactNotLineAfter(@Argument(ArgumentType.STRING) String start,
|
||||
@Argument(ArgumentType.TYPE) String asType,
|
||||
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
|
||||
@Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
|
||||
@Argument(ArgumentType.STRING) String reason) {
|
||||
|
||||
redactLineAfter(start, asType, ruleNumber, redactEverywhere, reason, null, false);
|
||||
|
||||
@ -440,9 +503,12 @@ public class Section {
|
||||
|
||||
@ThenAction
|
||||
@SuppressWarnings("unused")
|
||||
public void redactByRegEx(@Argument(ArgumentType.REGEX) String pattern, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
|
||||
@Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.TYPE) String asType, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
|
||||
@Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
|
||||
public void redactByRegEx(@Argument(ArgumentType.REGEX) String pattern,
|
||||
@Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
|
||||
@Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.TYPE) String asType,
|
||||
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
|
||||
@Argument(ArgumentType.STRING) String reason,
|
||||
@Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
|
||||
|
||||
redactByRegEx(pattern, patternCaseInsensitive, group, asType, ruleNumber, reason, legalBasis, true);
|
||||
}
|
||||
@ -450,8 +516,10 @@ public class Section {
|
||||
|
||||
@ThenAction
|
||||
@SuppressWarnings("unused")
|
||||
public void redactNotByRegEx(@Argument(ArgumentType.REGEX) String pattern, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
|
||||
@Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.TYPE) String asType, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
|
||||
public void redactNotByRegEx(@Argument(ArgumentType.REGEX) String pattern,
|
||||
@Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
|
||||
@Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.TYPE) String asType,
|
||||
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
|
||||
@Argument(ArgumentType.STRING) String reason) {
|
||||
|
||||
redactByRegEx(pattern, patternCaseInsensitive, group, asType, ruleNumber, reason, null, false);
|
||||
@ -460,9 +528,12 @@ public class Section {
|
||||
|
||||
@ThenAction
|
||||
@SuppressWarnings("unused")
|
||||
public void redactBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType,
|
||||
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
|
||||
@Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
|
||||
public void redactBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop,
|
||||
@Argument(ArgumentType.TYPE) String asType,
|
||||
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
|
||||
@Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
|
||||
@Argument(ArgumentType.STRING) String reason,
|
||||
@Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
|
||||
|
||||
redactBetween(start, stop, asType, ruleNumber, redactEverywhere, reason, legalBasis, true);
|
||||
}
|
||||
@ -470,8 +541,10 @@ public class Section {
|
||||
|
||||
@ThenAction
|
||||
@SuppressWarnings("unused")
|
||||
public void redactNotBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType,
|
||||
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
|
||||
public void redactNotBetween(@Argument(ArgumentType.STRING) String start,
|
||||
@Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType,
|
||||
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
|
||||
@Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
|
||||
@Argument(ArgumentType.STRING) String reason) {
|
||||
|
||||
redactBetween(start, stop, asType, ruleNumber, redactEverywhere, reason, null, false);
|
||||
@ -480,9 +553,13 @@ public class Section {
|
||||
|
||||
@ThenAction
|
||||
@SuppressWarnings("unused")
|
||||
public void redactLinesBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType,
|
||||
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
|
||||
@Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
|
||||
public void redactLinesBetween(@Argument(ArgumentType.STRING) String start,
|
||||
@Argument(ArgumentType.STRING) String stop,
|
||||
@Argument(ArgumentType.TYPE) String asType,
|
||||
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
|
||||
@Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
|
||||
@Argument(ArgumentType.STRING) String reason,
|
||||
@Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
|
||||
|
||||
redactLinesBetween(start, stop, asType, ruleNumber, redactEverywhere, reason, legalBasis, true);
|
||||
}
|
||||
@ -490,8 +567,11 @@ public class Section {
|
||||
|
||||
@ThenAction
|
||||
@SuppressWarnings("unused")
|
||||
public void redactNotLinesBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType,
|
||||
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
|
||||
public void redactNotLinesBetween(@Argument(ArgumentType.STRING) String start,
|
||||
@Argument(ArgumentType.STRING) String stop,
|
||||
@Argument(ArgumentType.TYPE) String asType,
|
||||
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
|
||||
@Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
|
||||
@Argument(ArgumentType.STRING) String reason) {
|
||||
|
||||
redactLinesBetween(start, stop, asType, ruleNumber, redactEverywhere, reason, null, false);
|
||||
@ -500,8 +580,10 @@ public class Section {
|
||||
|
||||
@ThenAction
|
||||
@SuppressWarnings("unused")
|
||||
public void redactCell(@Argument(ArgumentType.STRING) String cellHeader, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.TYPE) String type,
|
||||
@Argument(ArgumentType.BOOLEAN) boolean addAsRecommendations, @Argument(ArgumentType.STRING) String reason,
|
||||
public void redactCell(@Argument(ArgumentType.STRING) String cellHeader,
|
||||
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.TYPE) String type,
|
||||
@Argument(ArgumentType.BOOLEAN) boolean addAsRecommendations,
|
||||
@Argument(ArgumentType.STRING) String reason,
|
||||
@Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
|
||||
|
||||
annotateCell(cellHeader, ruleNumber, type, true, addAsRecommendations, reason, legalBasis);
|
||||
@ -510,8 +592,11 @@ public class Section {
|
||||
|
||||
@ThenAction
|
||||
@SuppressWarnings("unused")
|
||||
public void redactNotCell(@Argument(ArgumentType.STRING) String cellHeader, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.TYPE) String type,
|
||||
@Argument(ArgumentType.BOOLEAN) boolean addAsRecommendations, @Argument(ArgumentType.STRING) String reason) {
|
||||
public void redactNotCell(@Argument(ArgumentType.STRING) String cellHeader,
|
||||
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
|
||||
@Argument(ArgumentType.TYPE) String type,
|
||||
@Argument(ArgumentType.BOOLEAN) boolean addAsRecommendations,
|
||||
@Argument(ArgumentType.STRING) String reason) {
|
||||
|
||||
annotateCell(cellHeader, ruleNumber, type, false, addAsRecommendations, reason, null);
|
||||
}
|
||||
@ -519,9 +604,13 @@ public class Section {
|
||||
|
||||
@ThenAction
|
||||
@SuppressWarnings("unused")
|
||||
public void redactAndRecommendByRegEx(@Argument(ArgumentType.REGEX) String pattern, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
|
||||
@Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.TYPE) String asType, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
|
||||
@Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
|
||||
public void redactAndRecommendByRegEx(@Argument(ArgumentType.REGEX) String pattern,
|
||||
@Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
|
||||
@Argument(ArgumentType.INTEGER) int group,
|
||||
@Argument(ArgumentType.TYPE) String asType,
|
||||
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
|
||||
@Argument(ArgumentType.STRING) String reason,
|
||||
@Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
|
||||
|
||||
redactAndRecommendByRegEx(pattern, patternCaseInsensitive, group, asType, ruleNumber, reason, legalBasis, true);
|
||||
}
|
||||
@ -529,9 +618,12 @@ public class Section {
|
||||
|
||||
@ThenAction
|
||||
@SuppressWarnings("unused")
|
||||
public void redactNotAndRecommendByRegEx(@Argument(ArgumentType.REGEX) String pattern, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
|
||||
@Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.TYPE) String asType,
|
||||
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason) {
|
||||
public void redactNotAndRecommendByRegEx(@Argument(ArgumentType.REGEX) String pattern,
|
||||
@Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
|
||||
@Argument(ArgumentType.INTEGER) int group,
|
||||
@Argument(ArgumentType.TYPE) String asType,
|
||||
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
|
||||
@Argument(ArgumentType.STRING) String reason) {
|
||||
|
||||
redactAndRecommendByRegEx(pattern, patternCaseInsensitive, group, asType, ruleNumber, reason, null, false);
|
||||
}
|
||||
@ -539,8 +631,10 @@ public class Section {
|
||||
|
||||
@ThenAction
|
||||
@SuppressWarnings("unused")
|
||||
public void addRecommendationByRegEx(@Argument(ArgumentType.REGEX) String pattern, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
|
||||
@Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.TYPE) String asType) {
|
||||
public void addRecommendationByRegEx(@Argument(ArgumentType.REGEX) String pattern,
|
||||
@Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
|
||||
@Argument(ArgumentType.INTEGER) int group,
|
||||
@Argument(ArgumentType.TYPE) String asType) {
|
||||
|
||||
Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
|
||||
|
||||
@ -557,11 +651,14 @@ public class Section {
|
||||
|
||||
@ThenAction
|
||||
@SuppressWarnings("unused")
|
||||
public void redactNotAndReference(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.REFERENCE_TYPE) String referenceType,
|
||||
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason) {
|
||||
public void redactNotAndReference(@Argument(ArgumentType.TYPE) String type,
|
||||
@Argument(ArgumentType.REFERENCE_TYPE) String referenceType,
|
||||
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
|
||||
@Argument(ArgumentType.STRING) String reason) {
|
||||
|
||||
|
||||
Set<Entity> references = entities.stream().filter(entity -> entity.getType().equals(referenceType)).collect(Collectors.toSet());
|
||||
Set<Entity> references = entities.stream()
|
||||
.filter(entity -> entity.getType().equals(referenceType))
|
||||
.collect(Collectors.toSet());
|
||||
|
||||
entities.forEach(entity -> {
|
||||
if (entity.getType().equals(type)) {
|
||||
@ -576,8 +673,11 @@ public class Section {
|
||||
|
||||
@ThenAction
|
||||
@SuppressWarnings("unused")
|
||||
public void redactIfPrecededBy(@Argument(ArgumentType.STRING) String prefix, @Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
|
||||
@Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
|
||||
public void redactIfPrecededBy(@Argument(ArgumentType.STRING) String prefix,
|
||||
@Argument(ArgumentType.TYPE) String type,
|
||||
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
|
||||
@Argument(ArgumentType.STRING) String reason,
|
||||
@Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
|
||||
|
||||
entities.forEach(entity -> {
|
||||
if (entity.getType().equals(type) && searchText.indexOf(prefix + entity.getWord()) != 1) {
|
||||
@ -592,13 +692,16 @@ public class Section {
|
||||
|
||||
@ThenAction
|
||||
@SuppressWarnings("unused")
|
||||
public void addRedaction(@Argument(ArgumentType.STRING) String value, @Argument(ArgumentType.TYPE) String asType, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
|
||||
@Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
|
||||
public void addRedaction(@Argument(ArgumentType.STRING) String value, @Argument(ArgumentType.TYPE) String asType,
|
||||
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
|
||||
@Argument(ArgumentType.STRING) String reason,
|
||||
@Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
|
||||
|
||||
Set<Entity> found = findEntities(value.trim(), asType, true, true, ruleNumber, reason, legalBasis, Engine.RULE, false);
|
||||
EntitySearchUtils.addEntitiesIgnoreRank(entities, found);
|
||||
}
|
||||
|
||||
|
||||
@ThenAction
|
||||
@SuppressWarnings("unused")
|
||||
public void ignore(@Argument(ArgumentType.TYPE) String type) {
|
||||
@ -606,18 +709,22 @@ public class Section {
|
||||
entities.removeIf(entity -> entity.getType().equals(type) && entity.getEntityType().equals(EntityType.ENTITY));
|
||||
}
|
||||
|
||||
|
||||
@ThenAction
|
||||
@SuppressWarnings("unused")
|
||||
public void ignoreRecommendations(@Argument(ArgumentType.TYPE) String type) {
|
||||
|
||||
entities.removeIf(entity -> entity.getType().equals(type) && entity.getEntityType().equals(EntityType.RECOMMENDATION));
|
||||
entities.removeIf(entity -> entity.getType().equals(type) && entity.getEntityType()
|
||||
.equals(EntityType.RECOMMENDATION));
|
||||
}
|
||||
|
||||
|
||||
@ThenAction
|
||||
@SuppressWarnings("unused")
|
||||
public void expandToFalsePositiveByRegEx(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.STRING) String pattern,
|
||||
@Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, @Argument(ArgumentType.INTEGER) int group) {
|
||||
public void expandToFalsePositiveByRegEx(@Argument(ArgumentType.TYPE) String type,
|
||||
@Argument(ArgumentType.STRING) String pattern,
|
||||
@Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
|
||||
@Argument(ArgumentType.INTEGER) int group) {
|
||||
|
||||
Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
|
||||
|
||||
@ -646,8 +753,10 @@ public class Section {
|
||||
|
||||
@ThenAction
|
||||
@SuppressWarnings("unused")
|
||||
public void addHintAnnotationByRegEx(@Argument(ArgumentType.REGEX) String pattern, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
|
||||
@Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.TYPE) String asType) {
|
||||
public void addHintAnnotationByRegEx(@Argument(ArgumentType.REGEX) String pattern,
|
||||
@Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
|
||||
@Argument(ArgumentType.INTEGER) int group,
|
||||
@Argument(ArgumentType.TYPE) String asType) {
|
||||
|
||||
Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
|
||||
|
||||
@ -665,7 +774,8 @@ public class Section {
|
||||
|
||||
@ThenAction
|
||||
@SuppressWarnings("unused")
|
||||
public void addHintAnnotation(@Argument(ArgumentType.STRING) String value, @Argument(ArgumentType.TYPE) String asType) {
|
||||
public void addHintAnnotation(@Argument(ArgumentType.STRING) String value,
|
||||
@Argument(ArgumentType.TYPE) String asType) {
|
||||
|
||||
Set<Entity> found = findEntities(value.trim(), asType, true, false, 0, null, null, Engine.RULE, false);
|
||||
EntitySearchUtils.addEntitiesIgnoreRank(entities, found);
|
||||
@ -674,7 +784,8 @@ public class Section {
|
||||
|
||||
@ThenAction
|
||||
@SuppressWarnings("unused")
|
||||
public void recommendLineAfter(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.TYPE) String asType) {
|
||||
public void recommendLineAfter(@Argument(ArgumentType.STRING) String start,
|
||||
@Argument(ArgumentType.TYPE) String asType) {
|
||||
|
||||
String[] values = StringUtils.substringsBetween(text, start, "\n");
|
||||
|
||||
@ -699,7 +810,9 @@ public class Section {
|
||||
|
||||
@ThenAction
|
||||
@SuppressWarnings("unused")
|
||||
public void highlightCell(@Argument(ArgumentType.STRING) String cellHeader, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.TYPE) String type) {
|
||||
public void highlightCell(@Argument(ArgumentType.STRING) String cellHeader,
|
||||
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
|
||||
@Argument(ArgumentType.TYPE) String type) {
|
||||
|
||||
annotateCell(cellHeader, ruleNumber, type, false, false, null, null);
|
||||
}
|
||||
@ -707,7 +820,9 @@ public class Section {
|
||||
|
||||
@ThenAction
|
||||
@SuppressWarnings("unused")
|
||||
public void redactSection(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason,
|
||||
public void redactSection(@Argument(ArgumentType.TYPE) String type,
|
||||
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
|
||||
@Argument(ArgumentType.STRING) String reason,
|
||||
@Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
|
||||
|
||||
for (SectionArea sectionArea : sectionAreas) {
|
||||
@ -740,8 +855,8 @@ public class Section {
|
||||
}
|
||||
|
||||
|
||||
private void redactAndRecommendByRegEx(String pattern, boolean patternCaseInsensitive, int group, String asType, int ruleNumber, String reason, String legalBasis,
|
||||
boolean redaction) {
|
||||
private void redactAndRecommendByRegEx(String pattern, boolean patternCaseInsensitive, int group, String asType,
|
||||
int ruleNumber, String reason, String legalBasis, boolean redaction) {
|
||||
|
||||
Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
|
||||
Matcher matcher = compiledPattern.matcher(searchText);
|
||||
@ -756,11 +871,12 @@ public class Section {
|
||||
}
|
||||
|
||||
|
||||
private Set<Entity> findEntities(String value, String asType, boolean caseInsensitive, boolean redacted, int ruleNumber, String reason, String legalBasis, Engine engine, boolean asRecommendation) {
|
||||
private Set<Entity> findEntities(String value, String asType, boolean caseInsensitive, boolean redacted,
|
||||
int ruleNumber, String reason, String legalBasis, Engine engine,
|
||||
boolean asRecommendation) {
|
||||
|
||||
String text = caseInsensitive ? searchText.toLowerCase() : searchText;
|
||||
Set<Entity> found = EntitySearchUtils.findEntities(text, new SearchImplementation(value, caseInsensitive), dictionary.getType(asType),
|
||||
new FindEntityDetails(asType, headline, sectionNumber, false, false, engine, asRecommendation ? EntityType.RECOMMENDATION : EntityType.ENTITY));
|
||||
Set<Entity> found = EntitySearchUtils.findEntities(text, new SearchImplementation(value, caseInsensitive), dictionary.getType(asType), new FindEntityDetails(asType, headline, sectionNumber, false, false, engine, asRecommendation ? EntityType.RECOMMENDATION : EntityType.ENTITY));
|
||||
found.forEach(entity -> {
|
||||
if (redacted) {
|
||||
entity.setRedaction(true);
|
||||
@ -770,7 +886,26 @@ public class Section {
|
||||
}
|
||||
});
|
||||
|
||||
return EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary);
|
||||
return applyResizeRedactions(EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary));
|
||||
}
|
||||
|
||||
|
||||
private Set<Entity> applyResizeRedactions(Set<Entity> entitiesWithPositions) {
|
||||
|
||||
if (manualRedactions == null || manualRedactions.getResizeRedactions() == null || manualRedactions.getResizeRedactions().isEmpty()){
|
||||
return entitiesWithPositions;
|
||||
}
|
||||
|
||||
entitiesWithPositions.forEach(e -> e.getPositionSequences().forEach(pos -> {
|
||||
manualRedactions.getResizeRedactions().forEach(resize -> {
|
||||
if (resize.getAnnotationId().equals(pos.getId()) && resize.getValue().length() < e.getWord().length() && e.getWord().contains(resize.getValue())) {
|
||||
int start = e.getWord().indexOf(resize.getValue());
|
||||
e.setStart(e.getStart() + start);
|
||||
e.setEnd(e.getEnd() - resize.getValue().length());
|
||||
}
|
||||
});
|
||||
}));
|
||||
return entitiesWithPositions;
|
||||
}
|
||||
|
||||
|
||||
@ -800,7 +935,8 @@ public class Section {
|
||||
}
|
||||
|
||||
|
||||
private void annotateCell(String cellHeader, int ruleNumber, String type, boolean redact, boolean addAsRecommendations, String reason, String legalBasis) {
|
||||
private void annotateCell(String cellHeader, int ruleNumber, String type, boolean redact,
|
||||
boolean addAsRecommendations, String reason, String legalBasis) {
|
||||
|
||||
String cleanHeaderName = cellHeader.replaceAll("\n", "").replaceAll(" ", "").replaceAll("-", "");
|
||||
|
||||
@ -847,7 +983,8 @@ public class Section {
|
||||
}
|
||||
|
||||
|
||||
private void redactLineAfter(String start, String asType, int ruleNumber, boolean redactEverywhere, String reason, String legalBasis, boolean redaction) {
|
||||
private void redactLineAfter(String start, String asType, int ruleNumber, boolean redactEverywhere, String reason,
|
||||
String legalBasis, boolean redaction) {
|
||||
|
||||
String[] values = StringUtils.substringsBetween(text, start, "\n");
|
||||
|
||||
@ -866,7 +1003,8 @@ public class Section {
|
||||
}
|
||||
|
||||
|
||||
private void redactByRegEx(String pattern, boolean patternCaseInsensitive, int group, String asType, int ruleNumber, String reason, String legalBasis, boolean redaction) {
|
||||
private void redactByRegEx(String pattern, boolean patternCaseInsensitive, int group, String asType, int ruleNumber,
|
||||
String reason, String legalBasis, boolean redaction) {
|
||||
|
||||
Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
|
||||
|
||||
@ -882,7 +1020,8 @@ public class Section {
|
||||
}
|
||||
|
||||
|
||||
private void redactBetween(String start, String stop, String asType, int ruleNumber, boolean redactEverywhere, String reason, String legalBasis, boolean redaction) {
|
||||
private void redactBetween(String start, String stop, String asType, int ruleNumber, boolean redactEverywhere,
|
||||
String reason, String legalBasis, boolean redaction) {
|
||||
|
||||
String[] values = StringUtils.substringsBetween(searchText, start, stop);
|
||||
|
||||
@ -902,7 +1041,8 @@ public class Section {
|
||||
}
|
||||
|
||||
|
||||
private void redactLinesBetween(String start, String stop, String asType, int ruleNumber, boolean redactEverywhere, String reason, String legalBasis, boolean redaction) {
|
||||
private void redactLinesBetween(String start, String stop, String asType, int ruleNumber, boolean redactEverywhere,
|
||||
String reason, String legalBasis, boolean redaction) {
|
||||
|
||||
String[] values = StringUtils.substringsBetween(text, start, stop);
|
||||
|
||||
|
||||
@ -14,6 +14,7 @@ import org.kie.api.runtime.KieContainer;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization;
|
||||
import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
|
||||
@ -146,6 +147,7 @@ public class EntityRedactionService {
|
||||
.images(reanalysisSection.getImages())
|
||||
.sectionAreas(reanalysisSection.getSectionAreas())
|
||||
.fileAttributes(analyzeRequest.getFileAttributes())
|
||||
.manualRedactions(analyzeRequest.getManualRedactions())
|
||||
.build(), reanalysisSection.getSearchableText()));
|
||||
|
||||
}
|
||||
|
||||
@ -30,7 +30,9 @@ import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizati
|
||||
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
|
||||
import com.iqser.red.storage.commons.StorageAutoConfiguration;
|
||||
import com.iqser.red.storage.commons.service.StorageService;
|
||||
|
||||
import lombok.SneakyThrows;
|
||||
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.junit.After;
|
||||
import org.junit.Before;
|
||||
@ -216,16 +218,16 @@ public class RedactionIntegrationTest {
|
||||
}
|
||||
|
||||
|
||||
private void mockDictionaryCalls(Long version){
|
||||
private void mockDictionaryCalls(Long version) {
|
||||
|
||||
when(dictionaryClient.getDictionaryForType(VERTEBRATE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(VERTEBRATE, false));
|
||||
when(dictionaryClient.getDictionaryForType(ADDRESS + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(ADDRESS, false));
|
||||
when(dictionaryClient.getDictionaryForType(AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(AUTHOR, false));
|
||||
when(dictionaryClient.getDictionaryForType(SPONSOR + ":" + TEST_DOSSIER_TEMPLATE_ID,version)).thenReturn(getDictionaryResponse(SPONSOR, false));
|
||||
when(dictionaryClient.getDictionaryForType(SPONSOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(SPONSOR, false));
|
||||
when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(NO_REDACTION_INDICATOR, false));
|
||||
when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(REDACTION_INDICATOR, false));
|
||||
when(dictionaryClient.getDictionaryForType(HINT_ONLY + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(HINT_ONLY, false));
|
||||
when(dictionaryClient.getDictionaryForType(MUST_REDACT + ":" + TEST_DOSSIER_TEMPLATE_ID,version)).thenReturn(getDictionaryResponse(MUST_REDACT, false));
|
||||
when(dictionaryClient.getDictionaryForType(MUST_REDACT + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(MUST_REDACT, false));
|
||||
when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(PUBLISHED_INFORMATION, false));
|
||||
when(dictionaryClient.getDictionaryForType(TEST_METHOD + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(TEST_METHOD, false));
|
||||
when(dictionaryClient.getDictionaryForType(PII + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(PII, false));
|
||||
@ -240,6 +242,7 @@ public class RedactionIntegrationTest {
|
||||
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void test270Rotated() {
|
||||
|
||||
@ -493,14 +496,12 @@ public class RedactionIntegrationTest {
|
||||
deleted.add("David Chubb");
|
||||
deleted.add("mouse");
|
||||
|
||||
|
||||
reanlysisVersions.put("mouse", 3L);
|
||||
|
||||
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(3L);
|
||||
|
||||
when(dictionaryClient.getDictionaryForType(VERTEBRATE, null)).thenReturn(getDictionaryResponse(VERTEBRATE, false));
|
||||
|
||||
|
||||
start = System.currentTimeMillis();
|
||||
|
||||
ManualRedactions manualRedactions = new ManualRedactions();
|
||||
@ -671,7 +672,6 @@ public class RedactionIntegrationTest {
|
||||
|
||||
when(dictionaryClient.getDictionaryForType(VERTEBRATE, null)).thenReturn(getDictionaryResponse(VERTEBRATE, false));
|
||||
|
||||
|
||||
start = System.currentTimeMillis();
|
||||
|
||||
ManualRedactions manualRedactions = new ManualRedactions();
|
||||
@ -726,49 +726,48 @@ public class RedactionIntegrationTest {
|
||||
var colors = objectMapper.readValue(colorsResource.getInputStream(), Colors.class);
|
||||
|
||||
ClassPathResource typeResource = new ClassPathResource("colors/types.json");
|
||||
TypeReference<List<Type>> typeRefForTypes = new TypeReference<>(){};
|
||||
TypeReference<List<Type>> typeRefForTypes = new TypeReference<>() {
|
||||
};
|
||||
List<Type> types = objectMapper.readValue(typeResource.getInputStream(), typeRefForTypes);
|
||||
|
||||
|
||||
AnalyzeRequest request = prepareStorage("files/new/PublishedInformationTest.pdf");
|
||||
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
|
||||
ManualRedactions manualRedactions = new ManualRedactions();
|
||||
manualRedactions.getIdsToRemove().add(IdRemoval.builder()
|
||||
.annotationId("308dab9015bfafd911568cffe0a7f7de")
|
||||
.fileId(TEST_FILE_ID)
|
||||
.status(AnnotationStatus.APPROVED)
|
||||
.requestDate(OffsetDateTime.of(2022,05,23,8,30,07,475479, ZoneOffset.UTC))
|
||||
.processedDate(OffsetDateTime.of(2022,05,23,8,30,07,483651, ZoneOffset.UTC))
|
||||
.build());
|
||||
|
||||
manualRedactions.getForceRedactions().add(ManualForceRedaction.builder()
|
||||
.annotationId("0b56ea1a87c83f351df177315af94f0d")
|
||||
.fileId(TEST_FILE_ID)
|
||||
.status(AnnotationStatus.APPROVED)
|
||||
.requestDate(OffsetDateTime.of(2022,05,23,9,30,15,4653, ZoneOffset.UTC))
|
||||
.processedDate(OffsetDateTime.of(2022,05,23,9,30,15,794, ZoneOffset.UTC))
|
||||
.build());
|
||||
|
||||
manualRedactions.getIdsToRemove().add(IdRemoval.builder()
|
||||
.annotationId("0b56ea1a87c83f351df177315af94f0d")
|
||||
.fileId(TEST_FILE_ID)
|
||||
.status(AnnotationStatus.APPROVED)
|
||||
.requestDate(OffsetDateTime.of(2022,05,23,8,30,23,961721, ZoneOffset.UTC))
|
||||
.processedDate(OffsetDateTime.of(2022,05,23,8,30,23,96528, ZoneOffset.UTC))
|
||||
.build());
|
||||
manualRedactions.getIdsToRemove()
|
||||
.add(IdRemoval.builder()
|
||||
.annotationId("308dab9015bfafd911568cffe0a7f7de")
|
||||
.fileId(TEST_FILE_ID)
|
||||
.status(AnnotationStatus.APPROVED)
|
||||
.requestDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 07, 475479, ZoneOffset.UTC))
|
||||
.processedDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 07, 483651, ZoneOffset.UTC))
|
||||
.build());
|
||||
|
||||
manualRedactions.getForceRedactions()
|
||||
.add(ManualForceRedaction.builder()
|
||||
.annotationId("0b56ea1a87c83f351df177315af94f0d")
|
||||
.fileId(TEST_FILE_ID)
|
||||
.status(AnnotationStatus.APPROVED)
|
||||
.requestDate(OffsetDateTime.of(2022, 05, 23, 9, 30, 15, 4653, ZoneOffset.UTC))
|
||||
.processedDate(OffsetDateTime.of(2022, 05, 23, 9, 30, 15, 794, ZoneOffset.UTC))
|
||||
.build());
|
||||
|
||||
manualRedactions.getIdsToRemove()
|
||||
.add(IdRemoval.builder()
|
||||
.annotationId("0b56ea1a87c83f351df177315af94f0d")
|
||||
.fileId(TEST_FILE_ID)
|
||||
.status(AnnotationStatus.APPROVED)
|
||||
.requestDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 23, 961721, ZoneOffset.UTC))
|
||||
.processedDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 23, 96528, ZoneOffset.UTC))
|
||||
.build());
|
||||
|
||||
request.setManualRedactions(manualRedactions);
|
||||
|
||||
|
||||
AnalyzeResult result = analyzeService.analyze(request);
|
||||
|
||||
|
||||
AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
|
||||
.manualRedactions(manualRedactions)
|
||||
.colors(colors)
|
||||
.types(types)
|
||||
.manualRedactions(manualRedactions)
|
||||
.colors(colors)
|
||||
.types(types)
|
||||
.dossierId(TEST_DOSSIER_ID)
|
||||
.fileId(TEST_FILE_ID)
|
||||
.build());
|
||||
@ -808,6 +807,61 @@ public class RedactionIntegrationTest {
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testFindDictionaryEntryInResizedEntryPosition() throws IOException {
|
||||
|
||||
System.out.println("testResizeDictFound");
|
||||
|
||||
ClassPathResource colorsResource = new ClassPathResource("colors/colors.json");
|
||||
var colors = objectMapper.readValue(colorsResource.getInputStream(), Colors.class);
|
||||
|
||||
ClassPathResource typeResource = new ClassPathResource("colors/types.json");
|
||||
TypeReference<List<Type>> typeRefForTypes = new TypeReference<>() {
|
||||
};
|
||||
List<Type> types = objectMapper.readValue(typeResource.getInputStream(), typeRefForTypes);
|
||||
|
||||
long start = System.currentTimeMillis();
|
||||
|
||||
AnalyzeRequest request = prepareStorage("files/new/ResizeRedactionTestDoc.pdf");
|
||||
|
||||
ManualRedactions manualRedactions = new ManualRedactions();
|
||||
|
||||
manualRedactions.getResizeRedactions()
|
||||
.add(ManualResizeRedaction.builder()
|
||||
.annotationId("fd5ee82900beb7383f00480dea417569")
|
||||
.fileId(TEST_FILE_ID)
|
||||
.status(AnnotationStatus.APPROVED)
|
||||
.requestDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 07, 475479, ZoneOffset.UTC))
|
||||
.processedDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 07, 483651, ZoneOffset.UTC))
|
||||
.value("Regina")
|
||||
.positions(List.of(new Rectangle(147.86f, 225.9895f, 31.17696f, 10.048125f, 1)))
|
||||
.textBefore("Switzerland Contact point: ")
|
||||
.textAfter(" Dorn Phone:")
|
||||
.build());
|
||||
|
||||
request.setManualRedactions(manualRedactions);
|
||||
|
||||
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
|
||||
AnalyzeResult result = analyzeService.analyze(request);
|
||||
|
||||
AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
|
||||
.manualRedactions(manualRedactions)
|
||||
.colors(colors)
|
||||
.types(types)
|
||||
.dossierId(TEST_DOSSIER_ID)
|
||||
.fileId(TEST_FILE_ID)
|
||||
.build());
|
||||
|
||||
try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated.pdf")) {
|
||||
fileOutputStream.write(annotateResponse.getDocument());
|
||||
}
|
||||
long end = System.currentTimeMillis();
|
||||
|
||||
System.out.println("duration: " + (end - start));
|
||||
System.out.println("numberOfPages: " + result.getNumberOfPages());
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testManualRedaction() throws IOException {
|
||||
|
||||
@ -1607,15 +1661,16 @@ public class RedactionIntegrationTest {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testExpandByPrefixRegEx() throws IOException {
|
||||
|
||||
assertThat(dictionary.get(AUTHOR).contains("Robinson"));
|
||||
assertThat(! dictionary.get(AUTHOR).contains("Mrs. Robinson"));
|
||||
assertThat(!dictionary.get(AUTHOR).contains("Mrs. Robinson"));
|
||||
assertThat(dictionary.get(AUTHOR).contains("Bojangles"));
|
||||
assertThat(! dictionary.get(AUTHOR).contains("Mr. Bojangles"));
|
||||
assertThat(!dictionary.get(AUTHOR).contains("Mr. Bojangles"));
|
||||
assertThat(dictionary.get(AUTHOR).contains("Tambourine Man"));
|
||||
assertThat(! dictionary.get(AUTHOR).contains("Mr. Tambourine Man"));
|
||||
assertThat(!dictionary.get(AUTHOR).contains("Mr. Tambourine Man"));
|
||||
|
||||
String fileName = "files/mr-mrs.pdf";
|
||||
String outputFileName = OsUtils.getTemporaryDirectory() + "/Annotated.pdf";
|
||||
@ -1646,6 +1701,7 @@ public class RedactionIntegrationTest {
|
||||
assertThat(values).contains("Mr. Tambourine Man");
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
private AnalyzeRequest prepareStorage(InputStream stream) {
|
||||
|
||||
|
||||
@ -9,3 +9,4 @@ Naka-27 Aomachi, Nomi, Ishikawa 923-1101, Japan, JP
|
||||
Özgür U. Reyhan
|
||||
Sude Halide Nurullah
|
||||
Xinyi Y. Tao
|
||||
Dorn
|
||||
|
||||
Binary file not shown.
Loading…
x
Reference in New Issue
Block a user