Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RSS-107: Added funktion to redact Headline

Browse Source
This commit is contained in:
deiflaender 2022-09-13 12:09:09 +02:00
parent 5828e19422
commit d92757cda4
1 changed files with 105 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

158
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java
View File

@ -530,52 +530,72 @@ public class Section {
@Deprecated @Deprecated
@ThenAction @ThenAction
@SuppressWarnings("unused") @SuppressWarnings("unused")
public void redactBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType, public void redactBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop,
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere, @Argument(ArgumentType.TYPE) String asType,
@Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
@Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
redactBetween(start, stop,false, false, asType, ruleNumber, redactEverywhere, false, reason, legalBasis, true, false, false); @Argument(ArgumentType.STRING) String reason,
}
@ThenAction
@SuppressWarnings("unused")
public void redactBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType,
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
@Argument(ArgumentType.BOOLEAN) boolean excludeHeadLine, @Argument(ArgumentType.STRING) String reason,
@Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
redactBetween(start, stop,false,false, asType, ruleNumber, redactEverywhere, excludeHeadLine, reason, legalBasis, true, false, false); redactBetween(start, stop, false, false, asType, ruleNumber, redactEverywhere, false, reason, legalBasis, true, false, false);
} }
@ThenAction @ThenAction
@SuppressWarnings("unused") @SuppressWarnings("unused")
public void redactBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType, public void redactBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop,
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere, @Argument(ArgumentType.TYPE) String asType,
@Argument(ArgumentType.BOOLEAN) boolean excludeHeadLine, @Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
@Argument(ArgumentType.LEGAL_BASIS) String legalBasis, @Argument(ArgumentType.BOOLEAN) boolean skipRemoveEntitiesContainedInLarger, @Argument(ArgumentType.BOOLEAN) boolean sortedResult) { @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
@Argument(ArgumentType.BOOLEAN) boolean excludeHeadLine,
@Argument(ArgumentType.STRING) String reason,
@Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
redactBetween(start, stop,false, false, asType, ruleNumber, redactEverywhere, excludeHeadLine, reason, legalBasis, true, skipRemoveEntitiesContainedInLarger, sortedResult); redactBetween(start, stop, false, false, asType, ruleNumber, redactEverywhere, excludeHeadLine, reason, legalBasis, true, false, false);
} }
@ThenAction @ThenAction
@SuppressWarnings("unused") @SuppressWarnings("unused")
public void redactBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop,@Argument(ArgumentType.BOOLEAN) boolean includeStart, @Argument(ArgumentType.BOOLEAN) boolean includeStop, @Argument(ArgumentType.TYPE) String asType, public void redactBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop,
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere, @Argument(ArgumentType.TYPE) String asType,
@Argument(ArgumentType.BOOLEAN) boolean excludeHeadLine, @Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
@Argument(ArgumentType.LEGAL_BASIS) String legalBasis, @Argument(ArgumentType.BOOLEAN) boolean skipRemoveEntitiesContainedInLarger, @Argument(ArgumentType.BOOLEAN) boolean sortedResult) { @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
@Argument(ArgumentType.BOOLEAN) boolean excludeHeadLine,
@Argument(ArgumentType.STRING) String reason,
@Argument(ArgumentType.LEGAL_BASIS) String legalBasis,
@Argument(ArgumentType.BOOLEAN) boolean skipRemoveEntitiesContainedInLarger,
@Argument(ArgumentType.BOOLEAN) boolean sortedResult) {
redactBetween(start, stop,includeStart, includeStop, asType, ruleNumber, redactEverywhere, excludeHeadLine, reason, legalBasis, true, skipRemoveEntitiesContainedInLarger, sortedResult); redactBetween(start, stop, false, false, asType, ruleNumber, redactEverywhere, excludeHeadLine, reason, legalBasis, true, skipRemoveEntitiesContainedInLarger, sortedResult);
}
@ThenAction
@SuppressWarnings("unused")
public void redactBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop,
@Argument(ArgumentType.BOOLEAN) boolean includeStart,
@Argument(ArgumentType.BOOLEAN) boolean includeStop,
@Argument(ArgumentType.TYPE) String asType,
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
@Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
@Argument(ArgumentType.BOOLEAN) boolean excludeHeadLine,
@Argument(ArgumentType.STRING) String reason,
@Argument(ArgumentType.LEGAL_BASIS) String legalBasis,
@Argument(ArgumentType.BOOLEAN) boolean skipRemoveEntitiesContainedInLarger,
@Argument(ArgumentType.BOOLEAN) boolean sortedResult) {
redactBetween(start, stop, includeStart, includeStop, asType, ruleNumber, redactEverywhere, excludeHeadLine, reason, legalBasis, true, skipRemoveEntitiesContainedInLarger, sortedResult);
} }
@Deprecated @Deprecated
@ThenAction @ThenAction
@SuppressWarnings("unused") @SuppressWarnings("unused")
public void redactNotBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType, public void redactNotBetween(@Argument(ArgumentType.STRING) String start,
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere, @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType,
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
@Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
@Argument(ArgumentType.STRING) String reason) { @Argument(ArgumentType.STRING) String reason) {
redactBetween(start, stop, asType, ruleNumber, redactEverywhere, false, reason, null, false, false); redactBetween(start, stop, asType, ruleNumber, redactEverywhere, false, reason, null, false, false);
@ -584,9 +604,12 @@ public class Section {
@ThenAction @ThenAction
@SuppressWarnings("unused") @SuppressWarnings("unused")
public void redactNotBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType, public void redactNotBetween(@Argument(ArgumentType.STRING) String start,
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere, @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType,
@Argument(ArgumentType.BOOLEAN) boolean excludeHeadLine, @Argument(ArgumentType.STRING) String reason) { @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
@Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
@Argument(ArgumentType.BOOLEAN) boolean excludeHeadLine,
@Argument(ArgumentType.STRING) String reason) {
redactBetween(start, stop, asType, ruleNumber, redactEverywhere, excludeHeadLine, reason, null, false, false); redactBetween(start, stop, asType, ruleNumber, redactEverywhere, excludeHeadLine, reason, null, false, false);
} }
@ -594,20 +617,25 @@ public class Section {
@ThenAction @ThenAction
@SuppressWarnings("unused") @SuppressWarnings("unused")
public void redactLinesBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType, public void redactLinesBetween(@Argument(ArgumentType.STRING) String start,
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere, @Argument(ArgumentType.STRING) String stop,
@Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { @Argument(ArgumentType.TYPE) String asType,
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
@Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
@Argument(ArgumentType.STRING) String reason,
@Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
redactLinesBetween(start, stop, asType, ruleNumber, redactEverywhere, reason, legalBasis, true); redactLinesBetween(start, stop, asType, ruleNumber, redactEverywhere, reason, legalBasis, true);
} }
@ThenAction @ThenAction
@SuppressWarnings("unused") @SuppressWarnings("unused")
public void redactNotLinesBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType, public void redactNotLinesBetween(@Argument(ArgumentType.STRING) String start,
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere, @Argument(ArgumentType.STRING) String stop,
@Argument(ArgumentType.TYPE) String asType,
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
@Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
@Argument(ArgumentType.STRING) String reason) { @Argument(ArgumentType.STRING) String reason) {
redactLinesBetween(start, stop, asType, ruleNumber, redactEverywhere, reason, null, false); redactLinesBetween(start, stop, asType, ruleNumber, redactEverywhere, reason, null, false);
@ -750,7 +778,8 @@ public class Section {
@SuppressWarnings("unused") @SuppressWarnings("unused")
public void ignoreRecommendations(@Argument(ArgumentType.TYPE) String type) { public void ignoreRecommendations(@Argument(ArgumentType.TYPE) String type) {
entities.removeIf(entity -> entity.getType().equals(type) && entity.getEntityType().equals(EntityType.RECOMMENDATION)); entities.removeIf(entity -> entity.getType().equals(type) && entity.getEntityType()
.equals(EntityType.RECOMMENDATION));
} }
@ -855,7 +884,9 @@ public class Section {
@ThenAction @ThenAction
@SuppressWarnings("unused") @SuppressWarnings("unused")
public void redactSectionText(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason, public void redactSectionText(@Argument(ArgumentType.TYPE) String type,
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
@Argument(ArgumentType.STRING) String reason,
@Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
redactBetween("", "", type, ruleNumber, false, false, reason, legalBasis, true, false); redactBetween("", "", type, ruleNumber, false, false, reason, legalBasis, true, false);
@ -864,16 +895,34 @@ public class Section {
@ThenAction @ThenAction
@SuppressWarnings("unused") @SuppressWarnings("unused")
public void redactSectionTextWithoutHeadLine(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, public void redactSectionTextWithoutHeadLine(@Argument(ArgumentType.TYPE) String type,
@Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
@Argument(ArgumentType.STRING) String reason,
@Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
redactBetween("", "", type, ruleNumber, false, true, reason, legalBasis, true, false); redactBetween("", "", type, ruleNumber, false, true, reason, legalBasis, true, false);
} }
@ThenAction
public void redactHeadline(@Argument(ArgumentType.TYPE) String type,
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
@Argument(ArgumentType.STRING) String reason,
@Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
if (searchText.contains(headline)) {
Set<Entity> found = findEntities(headline.trim(), type, false, true, ruleNumber, reason, legalBasis, Engine.RULE, false);
EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
}
}
@ThenAction @ThenAction
@SuppressWarnings("unused") @SuppressWarnings("unused")
public void redactSection(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason, public void redactSection(@Argument(ArgumentType.TYPE) String type,
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
@Argument(ArgumentType.STRING) String reason,
@Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
for (SectionArea sectionArea : sectionAreas) { for (SectionArea sectionArea : sectionAreas) {
@ -906,8 +955,8 @@ public class Section {
} }
private void redactAndRecommendByRegEx(String pattern, boolean patternCaseInsensitive, int group, String asType, int ruleNumber, String reason, String legalBasis, private void redactAndRecommendByRegEx(String pattern, boolean patternCaseInsensitive, int group, String asType,
boolean redaction) { int ruleNumber, String reason, String legalBasis, boolean redaction) {
Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive); Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
Matcher matcher = compiledPattern.matcher(searchText); Matcher matcher = compiledPattern.matcher(searchText);
@ -1037,10 +1086,8 @@ public class Section {
} }
public void redactLineAfterAcrossColumns(String start, String asType, int ruleNumber, boolean redactEverywhere,
public void redactLineAfterAcrossColumns(String start, String asType, int ruleNumber, boolean redactEverywhere, String reason, String reason, String legalBasis) {
String legalBasis) {
String[] values = StringUtils.substringsBetween(searchableText.getAsStringWithLinebreaksSorted(), start, "\n"); String[] values = StringUtils.substringsBetween(searchableText.getAsStringWithLinebreaksSorted(), start, "\n");
@ -1059,7 +1106,6 @@ public class Section {
} }
private void redactByRegEx(String pattern, boolean patternCaseInsensitive, int group, String asType, int ruleNumber, private void redactByRegEx(String pattern, boolean patternCaseInsensitive, int group, String asType, int ruleNumber,
String reason, String legalBasis, boolean redaction) { String reason, String legalBasis, boolean redaction) {
@ -1077,8 +1123,10 @@ public class Section {
} }
private void redactBetween(String start, String stop,boolean includeStart, boolean includeStop, String asType, int ruleNumber, boolean redactEverywhere, boolean excludeHeadLine, String reason, String legalBasis, private void redactBetween(String start, String stop, boolean includeStart, boolean includeStop, String asType,
boolean redaction, boolean skipRemoveEntitiesContainedInLarger, boolean sortedResult) { int ruleNumber, boolean redactEverywhere, boolean excludeHeadLine, String reason,
String legalBasis, boolean redaction, boolean skipRemoveEntitiesContainedInLarger,
boolean sortedResult) {
String[] values = new String[1]; String[] values = new String[1];
@ -1106,15 +1154,19 @@ public class Section {
searchString = start + searchString; searchString = start + searchString;
} }
if(!stop.isEmpty() && includeStop){ if (!stop.isEmpty() && includeStop) {
searchString = searchString + stop; searchString = searchString + stop;
} }
Set<Entity> found = findEntities(searchString.trim(), asType, false, redaction, ruleNumber, reason, legalBasis, Engine.RULE, false); Set<Entity> found = findEntities(searchString.trim(), asType, false, redaction, ruleNumber, reason, legalBasis, Engine.RULE, false);
found.forEach(f -> { found.forEach(f -> {
f.setSkipRemoveEntitiesContainedInLarger(skipRemoveEntitiesContainedInLarger); f.setSkipRemoveEntitiesContainedInLarger(skipRemoveEntitiesContainedInLarger);
if(sortedResult){ if (sortedResult) {
f.setWord(searchableText.getAsStringWithLinebreaksSorted(f.getPositionSequences().stream().map(EntityPositionSequence::getSequences).flatMap(Collection::stream).collect(Collectors.toList()))); f.setWord(searchableText.getAsStringWithLinebreaksSorted(f.getPositionSequences()
.stream()
.map(EntityPositionSequence::getSequences)
.flatMap(Collection::stream)
.collect(Collectors.toList())));
} }
}); });