Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RSS-2: Section: added method redactSectionText and extended redactBetween(Not) with excludeHeadline flag

Browse Source
This commit is contained in:
Philipp Schramm 2022-09-08 12:10:07 +02:00
parent c3dadd6906
commit f1b5d605cc
1 changed files with 85 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

131
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java
View File

@ -1,5 +1,23 @@
package com.iqser.red.service.redaction.v1.server.redaction.model; package com.iqser.red.service.redaction.v1.server.redaction.model;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions; import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
import com.iqser.red.service.redaction.v1.model.ArgumentType; import com.iqser.red.service.redaction.v1.model.ArgumentType;
import com.iqser.red.service.redaction.v1.model.Engine; import com.iqser.red.service.redaction.v1.model.Engine;
@ -10,19 +28,10 @@ import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUti
import com.iqser.red.service.redaction.v1.server.redaction.utils.FindEntityDetails; import com.iqser.red.service.redaction.v1.server.redaction.utils.FindEntityDetails;
import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns; import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns;
import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation; import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
import lombok.Builder; import lombok.Builder;
import lombok.Data; import lombok.Data;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
@Data @Data
@Slf4j @Slf4j
@ -518,40 +527,54 @@ public class Section {
} }
@Deprecated
@ThenAction @ThenAction
@SuppressWarnings("unused") @SuppressWarnings("unused")
public void redactBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop, public void redactBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType,
@Argument(ArgumentType.TYPE) String asType, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
@Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
@Argument(ArgumentType.STRING) String reason, redactBetween(start, stop, asType, ruleNumber, redactEverywhere, false, reason, legalBasis, true);
}
@ThenAction
@SuppressWarnings("unused")
public void redactBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType,
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
@Argument(ArgumentType.BOOLEAN) boolean excludeHeadLine, @Argument(ArgumentType.STRING) String reason,
@Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
redactBetween(start, stop, asType, ruleNumber, redactEverywhere, reason, legalBasis, true); redactBetween(start, stop, asType, ruleNumber, redactEverywhere, excludeHeadLine, reason, legalBasis, true);
} }
@Deprecated
@ThenAction @ThenAction
@SuppressWarnings("unused") @SuppressWarnings("unused")
public void redactNotBetween(@Argument(ArgumentType.STRING) String start, public void redactNotBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType,
@Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
@Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
@Argument(ArgumentType.STRING) String reason) { @Argument(ArgumentType.STRING) String reason) {
redactBetween(start, stop, asType, ruleNumber, redactEverywhere, reason, null, false); redactBetween(start, stop, asType, ruleNumber, redactEverywhere, false, reason, null, false);
} }
@ThenAction @ThenAction
@SuppressWarnings("unused") @SuppressWarnings("unused")
public void redactLinesBetween(@Argument(ArgumentType.STRING) String start, public void redactNotBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType,
@Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
@Argument(ArgumentType.TYPE) String asType, @Argument(ArgumentType.BOOLEAN) boolean excludeHeadLine, @Argument(ArgumentType.STRING) String reason) {
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
@Argument(ArgumentType.BOOLEAN) boolean redactEverywhere, redactBetween(start, stop, asType, ruleNumber, redactEverywhere, excludeHeadLine, reason, null, false);
@Argument(ArgumentType.STRING) String reason, }
@Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
@ThenAction
@SuppressWarnings("unused")
public void redactLinesBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType,
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
@Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
redactLinesBetween(start, stop, asType, ruleNumber, redactEverywhere, reason, legalBasis, true); redactLinesBetween(start, stop, asType, ruleNumber, redactEverywhere, reason, legalBasis, true);
} }
@ -559,11 +582,8 @@ public class Section {
@ThenAction @ThenAction
@SuppressWarnings("unused") @SuppressWarnings("unused")
public void redactNotLinesBetween(@Argument(ArgumentType.STRING) String start, public void redactNotLinesBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType,
@Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
@Argument(ArgumentType.TYPE) String asType,
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
@Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
@Argument(ArgumentType.STRING) String reason) { @Argument(ArgumentType.STRING) String reason) {
redactLinesBetween(start, stop, asType, ruleNumber, redactEverywhere, reason, null, false); redactLinesBetween(start, stop, asType, ruleNumber, redactEverywhere, reason, null, false);
@ -706,8 +726,7 @@ public class Section {
@SuppressWarnings("unused") @SuppressWarnings("unused")
public void ignoreRecommendations(@Argument(ArgumentType.TYPE) String type) { public void ignoreRecommendations(@Argument(ArgumentType.TYPE) String type) {
entities.removeIf(entity -> entity.getType().equals(type) && entity.getEntityType() entities.removeIf(entity -> entity.getType().equals(type) && entity.getEntityType().equals(EntityType.RECOMMENDATION));
.equals(EntityType.RECOMMENDATION));
} }
@ -812,9 +831,25 @@ public class Section {
@ThenAction @ThenAction
@SuppressWarnings("unused") @SuppressWarnings("unused")
public void redactSection(@Argument(ArgumentType.TYPE) String type, public void redactSectionText(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason,
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
@Argument(ArgumentType.STRING) String reason,
redactBetween("", "", type, ruleNumber, false, false, reason, legalBasis, true);
}
@ThenAction
@SuppressWarnings("unused")
public void redactSectionTextWithoutHeadLine(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
@Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
redactBetween("", "", type, ruleNumber, false, true, reason, legalBasis, true);
}
@ThenAction
@SuppressWarnings("unused")
public void redactSection(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason,
@Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
for (SectionArea sectionArea : sectionAreas) { for (SectionArea sectionArea : sectionAreas) {
@ -847,8 +882,8 @@ public class Section {
} }
private void redactAndRecommendByRegEx(String pattern, boolean patternCaseInsensitive, int group, String asType, private void redactAndRecommendByRegEx(String pattern, boolean patternCaseInsensitive, int group, String asType, int ruleNumber, String reason, String legalBasis,
int ruleNumber, String reason, String legalBasis, boolean redaction) { boolean redaction) {
Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive); Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
Matcher matcher = compiledPattern.matcher(searchText); Matcher matcher = compiledPattern.matcher(searchText);
@ -995,16 +1030,20 @@ public class Section {
} }
private void redactBetween(String start, String stop, String asType, int ruleNumber, boolean redactEverywhere, private void redactBetween(String start, String stop, String asType, int ruleNumber, boolean redactEverywhere, boolean excludeHeadLine, String reason, String legalBasis,
String reason, String legalBasis, boolean redaction) { boolean redaction) {
String[] values = new String[1]; String[] values = new String[1];
if(start.isEmpty() && stop.isEmpty()){ if (start.isEmpty() && stop.isEmpty()) {
values[0] = searchText; if (excludeHeadLine) {
} else if(start.isEmpty() && searchText.contains(stop)){ values[0] = StringUtils.substringAfter(searchText, headline);
} else {
values[0] = searchText;
}
} else if (start.isEmpty() && searchText.contains(stop)) {
values[0] = StringUtils.substringBefore(searchText, stop); values[0] = StringUtils.substringBefore(searchText, stop);
} else if (stop.isEmpty() && searchText.contains(start)){ } else if (stop.isEmpty() && searchText.contains(start)) {
values[0] = StringUtils.substringAfter(searchText, start); values[0] = StringUtils.substringAfter(searchText, start);
} else { } else {
values = StringUtils.substringsBetween(searchText, start, stop); values = StringUtils.substringsBetween(searchText, start, stop);