Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-4064: Fixed find dictionary entries at position where rules entries are resized to smaller entries

Browse Source
This commit is contained in:
deiflaender 2022-06-02 10:32:48 +02:00
parent 50b2908763
commit 778c975649
5 changed files with 329 additions and 130 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

322
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java
View File

@ -18,6 +18,7 @@ import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
import com.iqser.red.service.redaction.v1.model.ArgumentType; import com.iqser.red.service.redaction.v1.model.ArgumentType;
import com.iqser.red.service.redaction.v1.model.Engine; import com.iqser.red.service.redaction.v1.model.Engine;
import com.iqser.red.service.redaction.v1.model.FileAttribute; import com.iqser.red.service.redaction.v1.model.FileAttribute;
@ -73,12 +74,16 @@ public class Section {
@Builder.Default @Builder.Default
private List<SectionArea> sectionAreas = new ArrayList<>(); private List<SectionArea> sectionAreas = new ArrayList<>();
private ManualRedactions manualRedactions;
@SuppressWarnings("unused") @SuppressWarnings("unused")
@WhenCondition @WhenCondition
public void addAiEntities(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.TYPE) String asType) { public void addAiEntities(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.TYPE) String asType) {
Set<Entity> entitiesOfType = nerEntities.stream().filter(nerEntity -> nerEntity.getType().equals(type)).collect(Collectors.toSet()); Set<Entity> entitiesOfType = nerEntities.stream()
.filter(nerEntity -> nerEntity.getType().equals(type))
.collect(Collectors.toSet());
List<String> values = entitiesOfType.stream().map(Entity::getWord).collect(Collectors.toList()); List<String> values = entitiesOfType.stream().map(Entity::getWord).collect(Collectors.toList());
Set<Entity> found = EntitySearchUtils.findEntities(searchText, new SearchImplementation(values, dictionary.isCaseInsensitiveDictionary(asType)), dictionary.getType(asType), new FindEntityDetails(asType, headline, sectionNumber, false, false, Engine.NER, EntityType.RECOMMENDATION)); Set<Entity> found = EntitySearchUtils.findEntities(searchText, new SearchImplementation(values, dictionary.isCaseInsensitiveDictionary(asType)), dictionary.getType(asType), new FindEntityDetails(asType, headline, sectionNumber, false, false, Engine.NER, EntityType.RECOMMENDATION));
EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary); EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary);
@ -105,15 +110,21 @@ public class Section {
nerEntities.removeAll(entitiesOfType); nerEntities.removeAll(entitiesOfType);
} }
@SuppressWarnings("unused") @SuppressWarnings("unused")
@WhenCondition @WhenCondition
public void combineAiTypes(@Argument(ArgumentType.TYPE) String startType, @Argument(ArgumentType.TYPE) String combineTypes, public void combineAiTypes(@Argument(ArgumentType.TYPE) String startType,
@Argument(ArgumentType.INTEGER) int maxDistanceBetween, @Argument(ArgumentType.TYPE) String asType, @Argument(ArgumentType.TYPE) String combineTypes,
@Argument(ArgumentType.INTEGER) int minPartMatches, @Argument(ArgumentType.BOOLEAN) boolean allowDuplicateTypes) { @Argument(ArgumentType.INTEGER) int maxDistanceBetween,
@Argument(ArgumentType.TYPE) String asType,
@Argument(ArgumentType.INTEGER) int minPartMatches,
@Argument(ArgumentType.BOOLEAN) boolean allowDuplicateTypes) {
Set<String> combineSet = Set.of(combineTypes.split(",")); Set<String> combineSet = Set.of(combineTypes.split(","));
List<Entity> sorted = nerEntities.stream().sorted(Comparator.comparing(Entity::getStart)).collect(Collectors.toList()); List<Entity> sorted = nerEntities.stream()
.sorted(Comparator.comparing(Entity::getStart))
.collect(Collectors.toList());
Set<Entity> found = new HashSet<>(); Set<Entity> found = new HashSet<>();
int start = -1; int start = -1;
int lastEnd = -1; int lastEnd = -1;
@ -170,49 +181,67 @@ public class Section {
} }
} }
@SuppressWarnings("unused") @SuppressWarnings("unused")
@WhenCondition @WhenCondition
public boolean fileAttributeByIdEquals(@Argument(ArgumentType.FILE_ATTRIBUTE) String id, @Argument(ArgumentType.STRING) String value) { public boolean fileAttributeByIdEquals(@Argument(ArgumentType.FILE_ATTRIBUTE) String id,
@Argument(ArgumentType.STRING) String value) {
return fileAttributes != null && fileAttributes.stream().anyMatch(attribute -> id.equals(attribute.getId()) && value.equals(attribute.getValue())); return fileAttributes != null && fileAttributes.stream()
.anyMatch(attribute -> id.equals(attribute.getId()) && value.equals(attribute.getValue()));
} }
@SuppressWarnings("unused") @SuppressWarnings("unused")
@WhenCondition @WhenCondition
public boolean fileAttributeByPlaceholderEquals(@Argument(ArgumentType.FILE_ATTRIBUTE) String placeholder, @Argument(ArgumentType.STRING) String value) { public boolean fileAttributeByPlaceholderEquals(@Argument(ArgumentType.FILE_ATTRIBUTE) String placeholder,
@Argument(ArgumentType.STRING) String value) {
return fileAttributes != null && fileAttributes.stream().anyMatch(attribute -> placeholder.equals(attribute.getPlaceholder()) && value.equals(attribute.getValue())); return fileAttributes != null && fileAttributes.stream()
.anyMatch(attribute -> placeholder.equals(attribute.getPlaceholder()) && value.equals(attribute.getValue()));
} }
@SuppressWarnings("unused") @SuppressWarnings("unused")
@WhenCondition @WhenCondition
public boolean fileAttributeByLabelEquals(@Argument(ArgumentType.FILE_ATTRIBUTE) String label, @Argument(ArgumentType.STRING) String value) { public boolean fileAttributeByLabelEquals(@Argument(ArgumentType.FILE_ATTRIBUTE) String label,
@Argument(ArgumentType.STRING) String value) {
return fileAttributes != null && fileAttributes.stream().anyMatch(attribute -> label.equals(attribute.getLabel()) && value.equals(attribute.getValue())); return fileAttributes != null && fileAttributes.stream()
.anyMatch(attribute -> label.equals(attribute.getLabel()) && value.equals(attribute.getValue()));
} }
@SuppressWarnings("unused") @SuppressWarnings("unused")
@WhenCondition @WhenCondition
public boolean fileAttributeByIdEqualsIgnoreCase(@Argument(ArgumentType.FILE_ATTRIBUTE) String id, @Argument(ArgumentType.STRING) String value) { public boolean fileAttributeByIdEqualsIgnoreCase(@Argument(ArgumentType.FILE_ATTRIBUTE) String id,
@Argument(ArgumentType.STRING) String value) {
return fileAttributes != null && fileAttributes.stream().anyMatch(attribute -> id.equals(attribute.getId()) && value.equalsIgnoreCase(attribute.getValue())); return fileAttributes != null && fileAttributes.stream()
.anyMatch(attribute -> id.equals(attribute.getId()) && value.equalsIgnoreCase(attribute.getValue()));
} }
@SuppressWarnings("unused") @SuppressWarnings("unused")
@WhenCondition @WhenCondition
public boolean fileAttributeByPlaceholderEqualsIgnoreCase(@Argument(ArgumentType.FILE_ATTRIBUTE) String placeholder, @Argument(ArgumentType.STRING) String value) { public boolean fileAttributeByPlaceholderEqualsIgnoreCase(@Argument(ArgumentType.FILE_ATTRIBUTE) String placeholder,
@Argument(ArgumentType.STRING) String value) {
return fileAttributes != null && fileAttributes.stream() return fileAttributes != null && fileAttributes.stream()
.anyMatch(attribute -> placeholder.equals(attribute.getPlaceholder()) && value.equalsIgnoreCase(attribute.getValue())); .anyMatch(attribute -> placeholder.equals(attribute.getPlaceholder()) && value.equalsIgnoreCase(attribute.getValue()));
} }
@SuppressWarnings("unused") @SuppressWarnings("unused")
@WhenCondition @WhenCondition
public boolean fileAttributeByLabelEqualsIgnoreCase(@Argument(ArgumentType.FILE_ATTRIBUTE) String label, @Argument(ArgumentType.STRING) String value) { public boolean fileAttributeByLabelEqualsIgnoreCase(@Argument(ArgumentType.FILE_ATTRIBUTE) String label,
@Argument(ArgumentType.STRING) String value) {
return fileAttributes != null && fileAttributes.stream().anyMatch(attribute -> label.equals(attribute.getLabel()) && value.equalsIgnoreCase(attribute.getValue())); return fileAttributes != null && fileAttributes.stream()
.anyMatch(attribute -> label.equals(attribute.getLabel()) && value.equalsIgnoreCase(attribute.getValue()));
} }
@SuppressWarnings("unused") @SuppressWarnings("unused")
@WhenCondition @WhenCondition
public boolean hasTableHeader(@Argument(ArgumentType.STRING) String headerName) { public boolean hasTableHeader(@Argument(ArgumentType.STRING) String headerName) {
@ -221,6 +250,7 @@ public class Section {
return tabularData != null && tabularData.containsKey(cleanHeaderName); return tabularData != null && tabularData.containsKey(cleanHeaderName);
} }
@SuppressWarnings("unused") @SuppressWarnings("unused")
@WhenCondition @WhenCondition
public boolean aiMatchesType(@Argument(ArgumentType.TYPE) String type) { public boolean aiMatchesType(@Argument(ArgumentType.TYPE) String type) {
@ -228,6 +258,7 @@ public class Section {
return nerEntities.stream().anyMatch(entity -> !entity.isIgnored() && entity.getType().equals(type)); return nerEntities.stream().anyMatch(entity -> !entity.isIgnored() && entity.getType().equals(type));
} }
@SuppressWarnings("unused") @SuppressWarnings("unused")
@WhenCondition @WhenCondition
public boolean matchesType(@Argument(ArgumentType.TYPE) String type) { public boolean matchesType(@Argument(ArgumentType.TYPE) String type) {
@ -235,6 +266,7 @@ public class Section {
return entities.stream().anyMatch(entity -> !entity.isIgnored() && entity.getType().equals(type)); return entities.stream().anyMatch(entity -> !entity.isIgnored() && entity.getType().equals(type));
} }
@SuppressWarnings("unused") @SuppressWarnings("unused")
@WhenCondition @WhenCondition
public boolean matchesImageType(@Argument(ArgumentType.TYPE) String type) { public boolean matchesImageType(@Argument(ArgumentType.TYPE) String type) {
@ -242,6 +274,7 @@ public class Section {
return images.stream().anyMatch(image -> !image.isIgnored() && image.getType().equals(type)); return images.stream().anyMatch(image -> !image.isIgnored() && image.getType().equals(type));
} }
@SuppressWarnings("unused") @SuppressWarnings("unused")
@WhenCondition @WhenCondition
public boolean headlineContainsWord(@Argument(ArgumentType.STRING) String word) { public boolean headlineContainsWord(@Argument(ArgumentType.STRING) String word) {
@ -249,9 +282,11 @@ public class Section {
return StringUtils.containsIgnoreCase(headline, word); return StringUtils.containsIgnoreCase(headline, word);
} }
@SuppressWarnings("unused") @SuppressWarnings("unused")
@WhenCondition @WhenCondition
public boolean containsRegEx(@Argument(ArgumentType.STRING) String regEx, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive) { public boolean containsRegEx(@Argument(ArgumentType.STRING) String regEx,
@Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive) {
var compiledPattern = Patterns.getCompiledPattern(regEx, patternCaseInsensitive); var compiledPattern = Patterns.getCompiledPattern(regEx, patternCaseInsensitive);
@ -260,19 +295,26 @@ public class Section {
return matcher.find(); return matcher.find();
} }
@SuppressWarnings("unused") @SuppressWarnings("unused")
@WhenCondition @WhenCondition
public boolean rowEquals(@Argument(ArgumentType.STRING) String headerName, @Argument(ArgumentType.STRING) String value) { public boolean rowEquals(@Argument(ArgumentType.STRING) String headerName,
@Argument(ArgumentType.STRING) String value) {
String cleanHeaderName = headerName.replaceAll("\n", "").replaceAll(" ", "").replaceAll("-", ""); String cleanHeaderName = headerName.replaceAll("\n", "").replaceAll(" ", "").replaceAll("-", "");
return tabularData != null && tabularData.containsKey(cleanHeaderName) && tabularData.get(cleanHeaderName).toString().equals(value); return tabularData != null && tabularData.containsKey(cleanHeaderName) && tabularData.get(cleanHeaderName)
.toString()
.equals(value);
} }
@ThenAction @ThenAction
@SuppressWarnings("unused") @SuppressWarnings("unused")
public void expandByPrefixRegEx(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.REGEX) String prefixPattern, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, @Argument(ArgumentType.INTEGER) int group) { public void expandByPrefixRegEx(@Argument(ArgumentType.TYPE) String type,
@Argument(ArgumentType.REGEX) String prefixPattern,
@Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
@Argument(ArgumentType.INTEGER) int group) {
expandByPrefixRegEx(type, prefixPattern, patternCaseInsensitive, group, null); expandByPrefixRegEx(type, prefixPattern, patternCaseInsensitive, group, null);
} }
@ -280,11 +322,15 @@ public class Section {
@ThenAction @ThenAction
@SuppressWarnings("unused") @SuppressWarnings("unused")
public void expandByPrefixRegEx(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.REGEX) String prefixPattern, public void expandByPrefixRegEx(@Argument(ArgumentType.TYPE) String type,
@Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, @Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.REGEX) String prefixPattern,
@Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
@Argument(ArgumentType.INTEGER) int group,
@Argument(ArgumentType.REGEX) String valuePattern) { @Argument(ArgumentType.REGEX) String valuePattern) {
if (StringUtils.isEmpty(prefixPattern)) return; if (StringUtils.isEmpty(prefixPattern)) {
return;
}
var compiledValuePattern = valuePattern == null ? null : Patterns.getCompiledPattern(valuePattern, patternCaseInsensitive); var compiledValuePattern = valuePattern == null ? null : Patterns.getCompiledPattern(valuePattern, patternCaseInsensitive);
var compiledPrefixPattern = Patterns.getCompiledPattern(prefixPattern, patternCaseInsensitive); var compiledPrefixPattern = Patterns.getCompiledPattern(prefixPattern, patternCaseInsensitive);
@ -329,8 +375,10 @@ public class Section {
@ThenAction @ThenAction
@SuppressWarnings("unused") @SuppressWarnings("unused")
public void expandByRegEx(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.REGEX) String suffixPattern, public void expandByRegEx(@Argument(ArgumentType.TYPE) String type,
@Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, @Argument(ArgumentType.INTEGER) int group) { @Argument(ArgumentType.REGEX) String suffixPattern,
@Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
@Argument(ArgumentType.INTEGER) int group) {
expandByRegEx(type, suffixPattern, patternCaseInsensitive, group, null); expandByRegEx(type, suffixPattern, patternCaseInsensitive, group, null);
} }
@ -338,11 +386,15 @@ public class Section {
@ThenAction @ThenAction
@SuppressWarnings("unused") @SuppressWarnings("unused")
public void expandByRegEx(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.REGEX) String suffixPattern, public void expandByRegEx(@Argument(ArgumentType.TYPE) String type,
@Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, @Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.REGEX) String suffixPattern,
@Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
@Argument(ArgumentType.INTEGER) int group,
@Argument(ArgumentType.REGEX) String valuePattern) { @Argument(ArgumentType.REGEX) String valuePattern) {
if (StringUtils.isEmpty(suffixPattern)) return; if (StringUtils.isEmpty(suffixPattern)) {
return;
}
var compiledValuePattern = valuePattern == null ? null : Patterns.getCompiledPattern(valuePattern, patternCaseInsensitive); var compiledValuePattern = valuePattern == null ? null : Patterns.getCompiledPattern(valuePattern, patternCaseInsensitive);
var compiledSuffixPattern = Patterns.getCompiledPattern(suffixPattern, patternCaseInsensitive); var compiledSuffixPattern = Patterns.getCompiledPattern(suffixPattern, patternCaseInsensitive);
@ -386,7 +438,9 @@ public class Section {
@ThenAction @ThenAction
@SuppressWarnings("unused") @SuppressWarnings("unused")
public void redactImage(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason, public void redactImage(@Argument(ArgumentType.TYPE) String type,
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
@Argument(ArgumentType.STRING) String reason,
@Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
redactImage(type, ruleNumber, reason, legalBasis, true); redactImage(type, ruleNumber, reason, legalBasis, true);
@ -395,7 +449,9 @@ public class Section {
@ThenAction @ThenAction
@SuppressWarnings("unused") @SuppressWarnings("unused")
public void redactNotImage(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason) { public void redactNotImage(@Argument(ArgumentType.TYPE) String type,
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
@Argument(ArgumentType.STRING) String reason) {
redactImage(type, ruleNumber, reason, null, false); redactImage(type, ruleNumber, reason, null, false);
} }
@ -403,7 +459,8 @@ public class Section {
@ThenAction @ThenAction
@SuppressWarnings("unused") @SuppressWarnings("unused")
public void redact(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason, public void redact(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
@Argument(ArgumentType.STRING) String reason,
@Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
redact(type, ruleNumber, reason, legalBasis, true); redact(type, ruleNumber, reason, legalBasis, true);
@ -412,7 +469,8 @@ public class Section {
@ThenAction @ThenAction
@SuppressWarnings("unused") @SuppressWarnings("unused")
public void redactNot(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason) { public void redactNot(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
@Argument(ArgumentType.STRING) String reason) {
redact(type, ruleNumber, reason, null, false); redact(type, ruleNumber, reason, null, false);
} }
@ -420,8 +478,10 @@ public class Section {
@ThenAction @ThenAction
@SuppressWarnings("unused") @SuppressWarnings("unused")
public void redactLineAfter(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.TYPE) String asType, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, public void redactLineAfter(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.TYPE) String asType,
@Argument(ArgumentType.BOOLEAN) boolean redactEverywhere, @Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
@Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
@Argument(ArgumentType.STRING) String reason,
@Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
redactLineAfter(start, asType, ruleNumber, redactEverywhere, reason, legalBasis, true); redactLineAfter(start, asType, ruleNumber, redactEverywhere, reason, legalBasis, true);
@ -430,8 +490,11 @@ public class Section {
@ThenAction @ThenAction
@SuppressWarnings("unused") @SuppressWarnings("unused")
public void redactNotLineAfter(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.TYPE) String asType, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, public void redactNotLineAfter(@Argument(ArgumentType.STRING) String start,
@Argument(ArgumentType.BOOLEAN) boolean redactEverywhere, @Argument(ArgumentType.STRING) String reason) { @Argument(ArgumentType.TYPE) String asType,
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
@Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
@Argument(ArgumentType.STRING) String reason) {
redactLineAfter(start, asType, ruleNumber, redactEverywhere, reason, null, false); redactLineAfter(start, asType, ruleNumber, redactEverywhere, reason, null, false);
@ -440,9 +503,12 @@ public class Section {
@ThenAction @ThenAction
@SuppressWarnings("unused") @SuppressWarnings("unused")
public void redactByRegEx(@Argument(ArgumentType.REGEX) String pattern, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, public void redactByRegEx(@Argument(ArgumentType.REGEX) String pattern,
@Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.TYPE) String asType, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
@Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { @Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.TYPE) String asType,
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
@Argument(ArgumentType.STRING) String reason,
@Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
redactByRegEx(pattern, patternCaseInsensitive, group, asType, ruleNumber, reason, legalBasis, true); redactByRegEx(pattern, patternCaseInsensitive, group, asType, ruleNumber, reason, legalBasis, true);
} }
@ -450,8 +516,10 @@ public class Section {
@ThenAction @ThenAction
@SuppressWarnings("unused") @SuppressWarnings("unused")
public void redactNotByRegEx(@Argument(ArgumentType.REGEX) String pattern, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, public void redactNotByRegEx(@Argument(ArgumentType.REGEX) String pattern,
@Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.TYPE) String asType, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
@Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.TYPE) String asType,
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
@Argument(ArgumentType.STRING) String reason) { @Argument(ArgumentType.STRING) String reason) {
redactByRegEx(pattern, patternCaseInsensitive, group, asType, ruleNumber, reason, null, false); redactByRegEx(pattern, patternCaseInsensitive, group, asType, ruleNumber, reason, null, false);
@ -460,9 +528,12 @@ public class Section {
@ThenAction @ThenAction
@SuppressWarnings("unused") @SuppressWarnings("unused")
public void redactBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType, public void redactBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop,
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere, @Argument(ArgumentType.TYPE) String asType,
@Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
@Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
@Argument(ArgumentType.STRING) String reason,
@Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
redactBetween(start, stop, asType, ruleNumber, redactEverywhere, reason, legalBasis, true); redactBetween(start, stop, asType, ruleNumber, redactEverywhere, reason, legalBasis, true);
} }
@ -470,8 +541,10 @@ public class Section {
@ThenAction @ThenAction
@SuppressWarnings("unused") @SuppressWarnings("unused")
public void redactNotBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType, public void redactNotBetween(@Argument(ArgumentType.STRING) String start,
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere, @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType,
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
@Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
@Argument(ArgumentType.STRING) String reason) { @Argument(ArgumentType.STRING) String reason) {
redactBetween(start, stop, asType, ruleNumber, redactEverywhere, reason, null, false); redactBetween(start, stop, asType, ruleNumber, redactEverywhere, reason, null, false);
@ -480,9 +553,13 @@ public class Section {
@ThenAction @ThenAction
@SuppressWarnings("unused") @SuppressWarnings("unused")
public void redactLinesBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType, public void redactLinesBetween(@Argument(ArgumentType.STRING) String start,
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere, @Argument(ArgumentType.STRING) String stop,
@Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { @Argument(ArgumentType.TYPE) String asType,
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
@Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
@Argument(ArgumentType.STRING) String reason,
@Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
redactLinesBetween(start, stop, asType, ruleNumber, redactEverywhere, reason, legalBasis, true); redactLinesBetween(start, stop, asType, ruleNumber, redactEverywhere, reason, legalBasis, true);
} }
@ -490,8 +567,11 @@ public class Section {
@ThenAction @ThenAction
@SuppressWarnings("unused") @SuppressWarnings("unused")
public void redactNotLinesBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType, public void redactNotLinesBetween(@Argument(ArgumentType.STRING) String start,
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere, @Argument(ArgumentType.STRING) String stop,
@Argument(ArgumentType.TYPE) String asType,
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
@Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
@Argument(ArgumentType.STRING) String reason) { @Argument(ArgumentType.STRING) String reason) {
redactLinesBetween(start, stop, asType, ruleNumber, redactEverywhere, reason, null, false); redactLinesBetween(start, stop, asType, ruleNumber, redactEverywhere, reason, null, false);
@ -500,8 +580,10 @@ public class Section {
@ThenAction @ThenAction
@SuppressWarnings("unused") @SuppressWarnings("unused")
public void redactCell(@Argument(ArgumentType.STRING) String cellHeader, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.TYPE) String type, public void redactCell(@Argument(ArgumentType.STRING) String cellHeader,
@Argument(ArgumentType.BOOLEAN) boolean addAsRecommendations, @Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.TYPE) String type,
@Argument(ArgumentType.BOOLEAN) boolean addAsRecommendations,
@Argument(ArgumentType.STRING) String reason,
@Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
annotateCell(cellHeader, ruleNumber, type, true, addAsRecommendations, reason, legalBasis); annotateCell(cellHeader, ruleNumber, type, true, addAsRecommendations, reason, legalBasis);
@ -510,8 +592,11 @@ public class Section {
@ThenAction @ThenAction
@SuppressWarnings("unused") @SuppressWarnings("unused")
public void redactNotCell(@Argument(ArgumentType.STRING) String cellHeader, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.TYPE) String type, public void redactNotCell(@Argument(ArgumentType.STRING) String cellHeader,
@Argument(ArgumentType.BOOLEAN) boolean addAsRecommendations, @Argument(ArgumentType.STRING) String reason) { @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
@Argument(ArgumentType.TYPE) String type,
@Argument(ArgumentType.BOOLEAN) boolean addAsRecommendations,
@Argument(ArgumentType.STRING) String reason) {
annotateCell(cellHeader, ruleNumber, type, false, addAsRecommendations, reason, null); annotateCell(cellHeader, ruleNumber, type, false, addAsRecommendations, reason, null);
} }
@ -519,9 +604,13 @@ public class Section {
@ThenAction @ThenAction
@SuppressWarnings("unused") @SuppressWarnings("unused")
public void redactAndRecommendByRegEx(@Argument(ArgumentType.REGEX) String pattern, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, public void redactAndRecommendByRegEx(@Argument(ArgumentType.REGEX) String pattern,
@Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.TYPE) String asType, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
@Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { @Argument(ArgumentType.INTEGER) int group,
@Argument(ArgumentType.TYPE) String asType,
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
@Argument(ArgumentType.STRING) String reason,
@Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
redactAndRecommendByRegEx(pattern, patternCaseInsensitive, group, asType, ruleNumber, reason, legalBasis, true); redactAndRecommendByRegEx(pattern, patternCaseInsensitive, group, asType, ruleNumber, reason, legalBasis, true);
} }
@ -529,9 +618,12 @@ public class Section {
@ThenAction @ThenAction
@SuppressWarnings("unused") @SuppressWarnings("unused")
public void redactNotAndRecommendByRegEx(@Argument(ArgumentType.REGEX) String pattern, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, public void redactNotAndRecommendByRegEx(@Argument(ArgumentType.REGEX) String pattern,
@Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.TYPE) String asType, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason) { @Argument(ArgumentType.INTEGER) int group,
@Argument(ArgumentType.TYPE) String asType,
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
@Argument(ArgumentType.STRING) String reason) {
redactAndRecommendByRegEx(pattern, patternCaseInsensitive, group, asType, ruleNumber, reason, null, false); redactAndRecommendByRegEx(pattern, patternCaseInsensitive, group, asType, ruleNumber, reason, null, false);
} }
@ -539,8 +631,10 @@ public class Section {
@ThenAction @ThenAction
@SuppressWarnings("unused") @SuppressWarnings("unused")
public void addRecommendationByRegEx(@Argument(ArgumentType.REGEX) String pattern, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, public void addRecommendationByRegEx(@Argument(ArgumentType.REGEX) String pattern,
@Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.TYPE) String asType) { @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
@Argument(ArgumentType.INTEGER) int group,
@Argument(ArgumentType.TYPE) String asType) {
Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive); Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
@ -557,11 +651,14 @@ public class Section {
@ThenAction @ThenAction
@SuppressWarnings("unused") @SuppressWarnings("unused")
public void redactNotAndReference(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.REFERENCE_TYPE) String referenceType, public void redactNotAndReference(@Argument(ArgumentType.TYPE) String type,
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason) { @Argument(ArgumentType.REFERENCE_TYPE) String referenceType,
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
@Argument(ArgumentType.STRING) String reason) {
Set<Entity> references = entities.stream()
Set<Entity> references = entities.stream().filter(entity -> entity.getType().equals(referenceType)).collect(Collectors.toSet()); .filter(entity -> entity.getType().equals(referenceType))
.collect(Collectors.toSet());
entities.forEach(entity -> { entities.forEach(entity -> {
if (entity.getType().equals(type)) { if (entity.getType().equals(type)) {
@ -576,8 +673,11 @@ public class Section {
@ThenAction @ThenAction
@SuppressWarnings("unused") @SuppressWarnings("unused")
public void redactIfPrecededBy(@Argument(ArgumentType.STRING) String prefix, @Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, public void redactIfPrecededBy(@Argument(ArgumentType.STRING) String prefix,
@Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { @Argument(ArgumentType.TYPE) String type,
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
@Argument(ArgumentType.STRING) String reason,
@Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
entities.forEach(entity -> { entities.forEach(entity -> {
if (entity.getType().equals(type) && searchText.indexOf(prefix + entity.getWord()) != 1) { if (entity.getType().equals(type) && searchText.indexOf(prefix + entity.getWord()) != 1) {
@ -592,13 +692,16 @@ public class Section {
@ThenAction @ThenAction
@SuppressWarnings("unused") @SuppressWarnings("unused")
public void addRedaction(@Argument(ArgumentType.STRING) String value, @Argument(ArgumentType.TYPE) String asType, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, public void addRedaction(@Argument(ArgumentType.STRING) String value, @Argument(ArgumentType.TYPE) String asType,
@Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
@Argument(ArgumentType.STRING) String reason,
@Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
Set<Entity> found = findEntities(value.trim(), asType, true, true, ruleNumber, reason, legalBasis, Engine.RULE, false); Set<Entity> found = findEntities(value.trim(), asType, true, true, ruleNumber, reason, legalBasis, Engine.RULE, false);
EntitySearchUtils.addEntitiesIgnoreRank(entities, found); EntitySearchUtils.addEntitiesIgnoreRank(entities, found);
} }
@ThenAction @ThenAction
@SuppressWarnings("unused") @SuppressWarnings("unused")
public void ignore(@Argument(ArgumentType.TYPE) String type) { public void ignore(@Argument(ArgumentType.TYPE) String type) {
@ -606,18 +709,22 @@ public class Section {
entities.removeIf(entity -> entity.getType().equals(type) && entity.getEntityType().equals(EntityType.ENTITY)); entities.removeIf(entity -> entity.getType().equals(type) && entity.getEntityType().equals(EntityType.ENTITY));
} }
@ThenAction @ThenAction
@SuppressWarnings("unused") @SuppressWarnings("unused")
public void ignoreRecommendations(@Argument(ArgumentType.TYPE) String type) { public void ignoreRecommendations(@Argument(ArgumentType.TYPE) String type) {
entities.removeIf(entity -> entity.getType().equals(type) && entity.getEntityType().equals(EntityType.RECOMMENDATION)); entities.removeIf(entity -> entity.getType().equals(type) && entity.getEntityType()
.equals(EntityType.RECOMMENDATION));
} }
@ThenAction @ThenAction
@SuppressWarnings("unused") @SuppressWarnings("unused")
public void expandToFalsePositiveByRegEx(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.STRING) String pattern, public void expandToFalsePositiveByRegEx(@Argument(ArgumentType.TYPE) String type,
@Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, @Argument(ArgumentType.INTEGER) int group) { @Argument(ArgumentType.STRING) String pattern,
@Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
@Argument(ArgumentType.INTEGER) int group) {
Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive); Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
@ -646,8 +753,10 @@ public class Section {
@ThenAction @ThenAction
@SuppressWarnings("unused") @SuppressWarnings("unused")
public void addHintAnnotationByRegEx(@Argument(ArgumentType.REGEX) String pattern, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, public void addHintAnnotationByRegEx(@Argument(ArgumentType.REGEX) String pattern,
@Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.TYPE) String asType) { @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
@Argument(ArgumentType.INTEGER) int group,
@Argument(ArgumentType.TYPE) String asType) {
Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive); Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
@ -665,7 +774,8 @@ public class Section {
@ThenAction @ThenAction
@SuppressWarnings("unused") @SuppressWarnings("unused")
public void addHintAnnotation(@Argument(ArgumentType.STRING) String value, @Argument(ArgumentType.TYPE) String asType) { public void addHintAnnotation(@Argument(ArgumentType.STRING) String value,
@Argument(ArgumentType.TYPE) String asType) {
Set<Entity> found = findEntities(value.trim(), asType, true, false, 0, null, null, Engine.RULE, false); Set<Entity> found = findEntities(value.trim(), asType, true, false, 0, null, null, Engine.RULE, false);
EntitySearchUtils.addEntitiesIgnoreRank(entities, found); EntitySearchUtils.addEntitiesIgnoreRank(entities, found);
@ -674,7 +784,8 @@ public class Section {
@ThenAction @ThenAction
@SuppressWarnings("unused") @SuppressWarnings("unused")
public void recommendLineAfter(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.TYPE) String asType) { public void recommendLineAfter(@Argument(ArgumentType.STRING) String start,
@Argument(ArgumentType.TYPE) String asType) {
String[] values = StringUtils.substringsBetween(text, start, "\n"); String[] values = StringUtils.substringsBetween(text, start, "\n");
@ -699,7 +810,9 @@ public class Section {
@ThenAction @ThenAction
@SuppressWarnings("unused") @SuppressWarnings("unused")
public void highlightCell(@Argument(ArgumentType.STRING) String cellHeader, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.TYPE) String type) { public void highlightCell(@Argument(ArgumentType.STRING) String cellHeader,
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
@Argument(ArgumentType.TYPE) String type) {
annotateCell(cellHeader, ruleNumber, type, false, false, null, null); annotateCell(cellHeader, ruleNumber, type, false, false, null, null);
} }
@ -707,7 +820,9 @@ public class Section {
@ThenAction @ThenAction
@SuppressWarnings("unused") @SuppressWarnings("unused")
public void redactSection(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason, public void redactSection(@Argument(ArgumentType.TYPE) String type,
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
@Argument(ArgumentType.STRING) String reason,
@Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
for (SectionArea sectionArea : sectionAreas) { for (SectionArea sectionArea : sectionAreas) {
@ -740,8 +855,8 @@ public class Section {
} }
private void redactAndRecommendByRegEx(String pattern, boolean patternCaseInsensitive, int group, String asType, int ruleNumber, String reason, String legalBasis, private void redactAndRecommendByRegEx(String pattern, boolean patternCaseInsensitive, int group, String asType,
boolean redaction) { int ruleNumber, String reason, String legalBasis, boolean redaction) {
Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive); Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
Matcher matcher = compiledPattern.matcher(searchText); Matcher matcher = compiledPattern.matcher(searchText);
@ -756,11 +871,12 @@ public class Section {
} }
private Set<Entity> findEntities(String value, String asType, boolean caseInsensitive, boolean redacted, int ruleNumber, String reason, String legalBasis, Engine engine, boolean asRecommendation) { private Set<Entity> findEntities(String value, String asType, boolean caseInsensitive, boolean redacted,
int ruleNumber, String reason, String legalBasis, Engine engine,
boolean asRecommendation) {
String text = caseInsensitive ? searchText.toLowerCase() : searchText; String text = caseInsensitive ? searchText.toLowerCase() : searchText;
Set<Entity> found = EntitySearchUtils.findEntities(text, new SearchImplementation(value, caseInsensitive), dictionary.getType(asType), Set<Entity> found = EntitySearchUtils.findEntities(text, new SearchImplementation(value, caseInsensitive), dictionary.getType(asType), new FindEntityDetails(asType, headline, sectionNumber, false, false, engine, asRecommendation ? EntityType.RECOMMENDATION : EntityType.ENTITY));
new FindEntityDetails(asType, headline, sectionNumber, false, false, engine, asRecommendation ? EntityType.RECOMMENDATION : EntityType.ENTITY));
found.forEach(entity -> { found.forEach(entity -> {
if (redacted) { if (redacted) {
entity.setRedaction(true); entity.setRedaction(true);
@ -770,7 +886,26 @@ public class Section {
} }
}); });
return EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary); return applyResizeRedactions(EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary));
}
private Set<Entity> applyResizeRedactions(Set<Entity> entitiesWithPositions) {
if (manualRedactions == null || manualRedactions.getResizeRedactions() == null || manualRedactions.getResizeRedactions().isEmpty()){
return entitiesWithPositions;
}
entitiesWithPositions.forEach(e -> e.getPositionSequences().forEach(pos -> {
manualRedactions.getResizeRedactions().forEach(resize -> {
if (resize.getAnnotationId().equals(pos.getId()) && resize.getValue().length() < e.getWord().length() && e.getWord().contains(resize.getValue())) {
int start = e.getWord().indexOf(resize.getValue());
e.setStart(e.getStart() + start);
e.setEnd(e.getEnd() - resize.getValue().length());
}
});
}));
return entitiesWithPositions;
} }
@ -800,7 +935,8 @@ public class Section {
} }
private void annotateCell(String cellHeader, int ruleNumber, String type, boolean redact, boolean addAsRecommendations, String reason, String legalBasis) { private void annotateCell(String cellHeader, int ruleNumber, String type, boolean redact,
boolean addAsRecommendations, String reason, String legalBasis) {
String cleanHeaderName = cellHeader.replaceAll("\n", "").replaceAll(" ", "").replaceAll("-", ""); String cleanHeaderName = cellHeader.replaceAll("\n", "").replaceAll(" ", "").replaceAll("-", "");
@ -847,7 +983,8 @@ public class Section {
} }
private void redactLineAfter(String start, String asType, int ruleNumber, boolean redactEverywhere, String reason, String legalBasis, boolean redaction) { private void redactLineAfter(String start, String asType, int ruleNumber, boolean redactEverywhere, String reason,
String legalBasis, boolean redaction) {
String[] values = StringUtils.substringsBetween(text, start, "\n"); String[] values = StringUtils.substringsBetween(text, start, "\n");
@ -866,7 +1003,8 @@ public class Section {
} }
private void redactByRegEx(String pattern, boolean patternCaseInsensitive, int group, String asType, int ruleNumber, String reason, String legalBasis, boolean redaction) { private void redactByRegEx(String pattern, boolean patternCaseInsensitive, int group, String asType, int ruleNumber,
String reason, String legalBasis, boolean redaction) {
Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive); Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
@ -882,7 +1020,8 @@ public class Section {
} }
private void redactBetween(String start, String stop, String asType, int ruleNumber, boolean redactEverywhere, String reason, String legalBasis, boolean redaction) { private void redactBetween(String start, String stop, String asType, int ruleNumber, boolean redactEverywhere,
String reason, String legalBasis, boolean redaction) {
String[] values = StringUtils.substringsBetween(searchText, start, stop); String[] values = StringUtils.substringsBetween(searchText, start, stop);
@ -902,7 +1041,8 @@ public class Section {
} }
private void redactLinesBetween(String start, String stop, String asType, int ruleNumber, boolean redactEverywhere, String reason, String legalBasis, boolean redaction) { private void redactLinesBetween(String start, String stop, String asType, int ruleNumber, boolean redactEverywhere,
String reason, String legalBasis, boolean redaction) {
String[] values = StringUtils.substringsBetween(text, start, stop); String[] values = StringUtils.substringsBetween(text, start, stop);

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java
View File

@ -14,6 +14,7 @@ import org.kie.api.runtime.KieContainer;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus; import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval; import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization; import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization;
import com.iqser.red.service.redaction.v1.model.AnalyzeRequest; import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
@ -146,6 +147,7 @@ public class EntityRedactionService {
.images(reanalysisSection.getImages()) .images(reanalysisSection.getImages())
.sectionAreas(reanalysisSection.getSectionAreas()) .sectionAreas(reanalysisSection.getSectionAreas())
.fileAttributes(analyzeRequest.getFileAttributes()) .fileAttributes(analyzeRequest.getFileAttributes())
.manualRedactions(analyzeRequest.getManualRedactions())
.build(), reanalysisSection.getSearchableText())); .build(), reanalysisSection.getSearchableText()));
} }

80
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java
View File

@ -30,7 +30,9 @@ import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizati
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import com.iqser.red.storage.commons.StorageAutoConfiguration; import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService; import com.iqser.red.storage.commons.service.StorageService;
import lombok.SneakyThrows; import lombok.SneakyThrows;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.junit.After; import org.junit.After;
import org.junit.Before; import org.junit.Before;
@ -240,6 +242,7 @@ public class RedactionIntegrationTest {
} }
@Test @Test
public void test270Rotated() { public void test270Rotated() {
@ -493,14 +496,12 @@ public class RedactionIntegrationTest {
deleted.add("David Chubb"); deleted.add("David Chubb");
deleted.add("mouse"); deleted.add("mouse");
reanlysisVersions.put("mouse", 3L); reanlysisVersions.put("mouse", 3L);
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(3L); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(3L);
when(dictionaryClient.getDictionaryForType(VERTEBRATE, null)).thenReturn(getDictionaryResponse(VERTEBRATE, false)); when(dictionaryClient.getDictionaryForType(VERTEBRATE, null)).thenReturn(getDictionaryResponse(VERTEBRATE, false));
start = System.currentTimeMillis(); start = System.currentTimeMillis();
ManualRedactions manualRedactions = new ManualRedactions(); ManualRedactions manualRedactions = new ManualRedactions();
@ -671,7 +672,6 @@ public class RedactionIntegrationTest {
when(dictionaryClient.getDictionaryForType(VERTEBRATE, null)).thenReturn(getDictionaryResponse(VERTEBRATE, false)); when(dictionaryClient.getDictionaryForType(VERTEBRATE, null)).thenReturn(getDictionaryResponse(VERTEBRATE, false));
start = System.currentTimeMillis(); start = System.currentTimeMillis();
ManualRedactions manualRedactions = new ManualRedactions(); ManualRedactions manualRedactions = new ManualRedactions();
@ -726,14 +726,15 @@ public class RedactionIntegrationTest {
var colors = objectMapper.readValue(colorsResource.getInputStream(), Colors.class); var colors = objectMapper.readValue(colorsResource.getInputStream(), Colors.class);
ClassPathResource typeResource = new ClassPathResource("colors/types.json"); ClassPathResource typeResource = new ClassPathResource("colors/types.json");
TypeReference<List<Type>> typeRefForTypes = new TypeReference<>(){}; TypeReference<List<Type>> typeRefForTypes = new TypeReference<>() {
};
List<Type> types = objectMapper.readValue(typeResource.getInputStream(), typeRefForTypes); List<Type> types = objectMapper.readValue(typeResource.getInputStream(), typeRefForTypes);
AnalyzeRequest request = prepareStorage("files/new/PublishedInformationTest.pdf"); AnalyzeRequest request = prepareStorage("files/new/PublishedInformationTest.pdf");
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
ManualRedactions manualRedactions = new ManualRedactions(); ManualRedactions manualRedactions = new ManualRedactions();
manualRedactions.getIdsToRemove().add(IdRemoval.builder() manualRedactions.getIdsToRemove()
.add(IdRemoval.builder()
.annotationId("308dab9015bfafd911568cffe0a7f7de") .annotationId("308dab9015bfafd911568cffe0a7f7de")
.fileId(TEST_FILE_ID) .fileId(TEST_FILE_ID)
.status(AnnotationStatus.APPROVED) .status(AnnotationStatus.APPROVED)
@ -741,7 +742,8 @@ public class RedactionIntegrationTest {
.processedDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 07, 483651, ZoneOffset.UTC)) .processedDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 07, 483651, ZoneOffset.UTC))
.build()); .build());
manualRedactions.getForceRedactions().add(ManualForceRedaction.builder() manualRedactions.getForceRedactions()
.add(ManualForceRedaction.builder()
.annotationId("0b56ea1a87c83f351df177315af94f0d") .annotationId("0b56ea1a87c83f351df177315af94f0d")
.fileId(TEST_FILE_ID) .fileId(TEST_FILE_ID)
.status(AnnotationStatus.APPROVED) .status(AnnotationStatus.APPROVED)
@ -749,7 +751,8 @@ public class RedactionIntegrationTest {
.processedDate(OffsetDateTime.of(2022, 05, 23, 9, 30, 15, 794, ZoneOffset.UTC)) .processedDate(OffsetDateTime.of(2022, 05, 23, 9, 30, 15, 794, ZoneOffset.UTC))
.build()); .build());
manualRedactions.getIdsToRemove().add(IdRemoval.builder() manualRedactions.getIdsToRemove()
.add(IdRemoval.builder()
.annotationId("0b56ea1a87c83f351df177315af94f0d") .annotationId("0b56ea1a87c83f351df177315af94f0d")
.fileId(TEST_FILE_ID) .fileId(TEST_FILE_ID)
.status(AnnotationStatus.APPROVED) .status(AnnotationStatus.APPROVED)
@ -757,14 +760,10 @@ public class RedactionIntegrationTest {
.processedDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 23, 96528, ZoneOffset.UTC)) .processedDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 23, 96528, ZoneOffset.UTC))
.build()); .build());
request.setManualRedactions(manualRedactions); request.setManualRedactions(manualRedactions);
AnalyzeResult result = analyzeService.analyze(request); AnalyzeResult result = analyzeService.analyze(request);
AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder() AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
.manualRedactions(manualRedactions) .manualRedactions(manualRedactions)
.colors(colors) .colors(colors)
@ -808,6 +807,61 @@ public class RedactionIntegrationTest {
} }
@Test
public void testFindDictionaryEntryInResizedEntryPosition() throws IOException {
System.out.println("testResizeDictFound");
ClassPathResource colorsResource = new ClassPathResource("colors/colors.json");
var colors = objectMapper.readValue(colorsResource.getInputStream(), Colors.class);
ClassPathResource typeResource = new ClassPathResource("colors/types.json");
TypeReference<List<Type>> typeRefForTypes = new TypeReference<>() {
};
List<Type> types = objectMapper.readValue(typeResource.getInputStream(), typeRefForTypes);
long start = System.currentTimeMillis();
AnalyzeRequest request = prepareStorage("files/new/ResizeRedactionTestDoc.pdf");
ManualRedactions manualRedactions = new ManualRedactions();
manualRedactions.getResizeRedactions()
.add(ManualResizeRedaction.builder()
.annotationId("fd5ee82900beb7383f00480dea417569")
.fileId(TEST_FILE_ID)
.status(AnnotationStatus.APPROVED)
.requestDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 07, 475479, ZoneOffset.UTC))
.processedDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 07, 483651, ZoneOffset.UTC))
.value("Regina")
.positions(List.of(new Rectangle(147.86f, 225.9895f, 31.17696f, 10.048125f, 1)))
.textBefore("Switzerland Contact point: ")
.textAfter(" Dorn Phone:")
.build());
request.setManualRedactions(manualRedactions);
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
AnalyzeResult result = analyzeService.analyze(request);
AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
.manualRedactions(manualRedactions)
.colors(colors)
.types(types)
.dossierId(TEST_DOSSIER_ID)
.fileId(TEST_FILE_ID)
.build());
try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated.pdf")) {
fileOutputStream.write(annotateResponse.getDocument());
}
long end = System.currentTimeMillis();
System.out.println("duration: " + (end - start));
System.out.println("numberOfPages: " + result.getNumberOfPages());
}
@Test @Test
public void testManualRedaction() throws IOException { public void testManualRedaction() throws IOException {
@ -1607,6 +1661,7 @@ public class RedactionIntegrationTest {
} }
} }
@Test @Test
public void testExpandByPrefixRegEx() throws IOException { public void testExpandByPrefixRegEx() throws IOException {
@ -1646,6 +1701,7 @@ public class RedactionIntegrationTest {
assertThat(values).contains("Mr. Tambourine Man"); assertThat(values).contains("Mr. Tambourine Man");
} }
@SneakyThrows @SneakyThrows
private AnalyzeRequest prepareStorage(InputStream stream) { private AnalyzeRequest prepareStorage(InputStream stream) {

1
redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/PII.txt
View File

@ -9,3 +9,4 @@ Naka-27 Aomachi, Nomi, Ishikawa 923-1101, Japan, JP
Özgür U. Reyhan Özgür U. Reyhan
Sude Halide Nurullah Sude Halide Nurullah
Xinyi Y. Tao Xinyi Y. Tao
Dorn

BIN
redaction-service-v1/redaction-service-server-v1/src/test/resources/files/new/ResizeRedactionTestDoc.pdf Normal file
View File

Binary file not shown.