diff --git a/redaction-service-v1/redaction-service-api-v1/pom.xml b/redaction-service-v1/redaction-service-api-v1/pom.xml
index 5337144b..fa22253a 100644
--- a/redaction-service-v1/redaction-service-api-v1/pom.xml
+++ b/redaction-service-v1/redaction-service-api-v1/pom.xml
@@ -12,7 +12,7 @@
redaction-service-api-v1
- 0.103.0
+ 0.121.0
diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java
index 8967510d..b23d7b59 100644
--- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java
+++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java
@@ -26,6 +26,7 @@ public class RedactionLogEntry {
private String value;
private String reason;
private int matchedRule;
+ private boolean rectangle;
private String legalBasis;
private boolean redacted;
private boolean isHint;
diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java
index ac1b0dde..49b8f310 100644
--- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java
+++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java
@@ -1,26 +1,34 @@
package com.iqser.red.service.redaction.v1.server.redaction.model;
+import static com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary.RECOMMENDATION_PREFIX;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+import org.apache.commons.lang3.StringUtils;
+
import com.iqser.red.service.redaction.v1.model.ArgumentType;
import com.iqser.red.service.redaction.v1.model.Engine;
import com.iqser.red.service.redaction.v1.model.FileAttribute;
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns;
+
import lombok.Builder;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.StringUtils;
-
-import java.lang.annotation.ElementType;
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
-import java.lang.annotation.Target;
-import java.util.*;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import java.util.stream.Collectors;
-
-import static com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary.RECOMMENDATION_PREFIX;
@Data
@Slf4j
@@ -62,39 +70,69 @@ public class Section {
@WhenCondition
public boolean fileAttributeByIdEquals(@Argument(ArgumentType.FILE_ATTRIBUTE) String id,
@Argument(ArgumentType.STRING) String value) {
- return fileAttributes != null && fileAttributes.stream().filter(attribute -> id.equals(attribute.getId()) && value.equals(attribute.getValue())).findFirst().isPresent();
+
+ return fileAttributes != null && fileAttributes.stream()
+ .filter(attribute -> id.equals(attribute.getId()) && value.equals(attribute.getValue()))
+ .findFirst()
+ .isPresent();
}
+
@WhenCondition
public boolean fileAttributeByPlaceholderEquals(@Argument(ArgumentType.FILE_ATTRIBUTE) String placeholder,
@Argument(ArgumentType.STRING) String value) {
- return fileAttributes != null && fileAttributes.stream().filter(attribute -> placeholder.equals(attribute.getPlaceholder()) && value.equals(attribute.getValue())).findFirst().isPresent();
+
+ return fileAttributes != null && fileAttributes.stream()
+ .filter(attribute -> placeholder.equals(attribute.getPlaceholder()) && value.equals(attribute.getValue()))
+ .findFirst()
+ .isPresent();
}
+
@WhenCondition
public boolean fileAttributeByLabelEquals(@Argument(ArgumentType.FILE_ATTRIBUTE) String label,
@Argument(ArgumentType.STRING) String value) {
- return fileAttributes != null && fileAttributes.stream().filter(attribute -> label.equals(attribute.getLabel()) && value.equals(attribute.getValue())).findFirst().isPresent();
+
+ return fileAttributes != null && fileAttributes.stream()
+ .filter(attribute -> label.equals(attribute.getLabel()) && value.equals(attribute.getValue()))
+ .findFirst()
+ .isPresent();
}
+
@WhenCondition
public boolean fileAttributeByIdEqualsIgnoreCase(@Argument(ArgumentType.FILE_ATTRIBUTE) String id,
@Argument(ArgumentType.STRING) String value) {
- return fileAttributes != null && fileAttributes.stream().filter(attribute -> id.equals(attribute.getId()) && value.equalsIgnoreCase(attribute.getValue())).findFirst().isPresent();
+
+ return fileAttributes != null && fileAttributes.stream()
+ .filter(attribute -> id.equals(attribute.getId()) && value.equalsIgnoreCase(attribute.getValue()))
+ .findFirst()
+ .isPresent();
}
+
@WhenCondition
public boolean fileAttributeByPlaceholderEqualsIgnoreCase(@Argument(ArgumentType.FILE_ATTRIBUTE) String placeholder,
@Argument(ArgumentType.STRING) String value) {
- return fileAttributes != null && fileAttributes.stream().filter(attribute -> placeholder.equals(attribute.getPlaceholder()) && value.equalsIgnoreCase(attribute.getValue())).findFirst().isPresent();
+
+ return fileAttributes != null && fileAttributes.stream()
+ .filter(attribute -> placeholder.equals(attribute.getPlaceholder()) && value.equalsIgnoreCase(attribute.getValue()))
+ .findFirst()
+ .isPresent();
}
+
@WhenCondition
public boolean fileAttributeByLabelEqualsIgnoreCase(@Argument(ArgumentType.FILE_ATTRIBUTE) String label,
@Argument(ArgumentType.STRING) String value) {
- return fileAttributes != null && fileAttributes.stream().filter(attribute -> label.equals(attribute.getLabel()) && value.equalsIgnoreCase(attribute.getValue())).findFirst().isPresent();
+
+ return fileAttributes != null && fileAttributes.stream()
+ .filter(attribute -> label.equals(attribute.getLabel()) && value.equalsIgnoreCase(attribute.getValue()))
+ .findFirst()
+ .isPresent();
}
+
@WhenCondition
public boolean rowEquals(@Argument(ArgumentType.STRING) String headerName,
@Argument(ArgumentType.STRING) String value) {
@@ -106,6 +144,7 @@ public class Section {
.equals(value);
}
+
@WhenCondition
public boolean hasTableHeader(@Argument(ArgumentType.STRING) String headerName) {
@@ -113,18 +152,22 @@ public class Section {
return tabularData != null && tabularData.containsKey(cleanHeaderName);
}
+
@WhenCondition
public boolean matchesType(@Argument(ArgumentType.TYPE) String type) {
return entities.stream().anyMatch(entity -> entity.getType().equals(type));
}
+
@WhenCondition
public boolean matchesImageType(@Argument(ArgumentType.TYPE) String type) {
+
return images.stream().anyMatch(image -> image.getType().equals(type));
}
+
@WhenCondition
public boolean headlineContainsWord(@Argument(ArgumentType.STRING) String word) {
@@ -133,13 +176,14 @@ public class Section {
@ThenAction
- public void expandByRegEx(@Argument(ArgumentType.TYPE) String type,
- @Argument(ArgumentType.REGEX) String pattern,
+ public void expandByRegEx(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.REGEX) String pattern,
@Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
@Argument(ArgumentType.INTEGER) int group) {
+
expandByRegEx(type, pattern, patternCaseInsensitive, group, null);
}
+
@ThenAction
public void expandByRegEx(@Argument(ArgumentType.TYPE) String type,
@Argument(ArgumentType.REGEX) String suffixPattern,
@@ -162,7 +206,7 @@ public class Section {
continue;
}
- if(compiledValuePattern != null) {
+ if (compiledValuePattern != null) {
Matcher matcherCompiledValuePattern = compiledValuePat.matcher(entity.getWord());
if (!matcherCompiledValuePattern.find()) {
continue;
@@ -173,10 +217,10 @@ public class Section {
while (matcher.find()) {
String match = matcher.group(group);
- if (StringUtils.isNotBlank(match)) {
- expanded.addAll(findEntities(entity.getWord() + match, type, false, entity.isRedaction(), entity.getMatchedRule(), entity
- .getRedactionReason(), entity.getLegalBasis()));
+ if (StringUtils.isNotBlank(match)) {
+ Set expandedEntities = findEntities(entity.getWord() + match, type, false, entity.isRedaction(), entity.getMatchedRule(), entity.getRedactionReason(), entity.getLegalBasis());
+ expanded.addAll(EntitySearchUtils.findNonOverlappingMatchEntities(entities, expandedEntities));
}
}
}
@@ -185,6 +229,7 @@ public class Section {
EntitySearchUtils.removeEntitiesContainedInLarger(entities);
}
+
@ThenAction
public void redactImage(@Argument(ArgumentType.TYPE) String type,
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
@@ -201,9 +246,9 @@ public class Section {
});
}
+
@ThenAction
- public void redact(@Argument(ArgumentType.TYPE) String type,
- @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
+ public void redact(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
@Argument(ArgumentType.STRING) String reason,
@Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
@@ -220,6 +265,7 @@ public class Section {
});
}
+
@ThenAction
public void redactNotImage(@Argument(ArgumentType.TYPE) String type,
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
@@ -234,9 +280,9 @@ public class Section {
});
}
+
@ThenAction
- public void redactNot(@Argument(ArgumentType.TYPE) String type,
- @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
+ public void redactNot(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
@Argument(ArgumentType.STRING) String reason) {
boolean hasRecommendationDictionary = dictionaryTypes.contains(RECOMMENDATION_PREFIX + type);
@@ -252,15 +298,23 @@ public class Section {
}
+ public void ignore(String type) {
+
+ entities.removeIf(entity -> entity.getType().equals(type));
+ }
+
+
@ThenAction
public void redactNotAndReference(@Argument(ArgumentType.TYPE) String type,
@Argument(ArgumentType.REFERENCE_TYPE) String referenceType,
- @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
- @Argument(ArgumentType.STRING) String reason) {
+ @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
+ @Argument(ArgumentType.STRING) String reason) {
boolean hasRecommendationDictionary = dictionaryTypes.contains(RECOMMENDATION_PREFIX + type);
- Set references = entities.stream().filter(entity -> entity.getType().equals(referenceType)).collect(Collectors.toSet());
+ Set references = entities.stream()
+ .filter(entity -> entity.getType().equals(referenceType))
+ .collect(Collectors.toSet());
entities.forEach(entity -> {
if (entity.getType().equals(type) || hasRecommendationDictionary && entity.getType()
@@ -274,7 +328,6 @@ public class Section {
}
-
@ThenAction
public void expandToHintAnnotationByRegEx(@Argument(ArgumentType.TYPE) String type,
@Argument(ArgumentType.STRING) String pattern,
@@ -305,6 +358,7 @@ public class Section {
EntitySearchUtils.removeEntitiesContainedInLarger(entities);
}
+
@ThenAction
public void addHintAnnotationByRegEx(@Argument(ArgumentType.REGEX) String pattern,
@Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
@@ -324,6 +378,7 @@ public class Section {
}
}
+
@ThenAction
public void redactIfPrecededBy(@Argument(ArgumentType.STRING) String prefix,
@Argument(ArgumentType.TYPE) String type,
@@ -341,6 +396,7 @@ public class Section {
});
}
+
@ThenAction
public void addHintAnnotation(@Argument(ArgumentType.STRING) String value,
@Argument(ArgumentType.TYPE) String asType) {
@@ -349,9 +405,9 @@ public class Section {
EntitySearchUtils.addEntitiesIgnoreRank(entities, found);
}
+
@ThenAction
- public void addRedaction(@Argument(ArgumentType.STRING) String value,
- @Argument(ArgumentType.TYPE) String asType,
+ public void addRedaction(@Argument(ArgumentType.STRING) String value, @Argument(ArgumentType.TYPE) String asType,
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
@Argument(ArgumentType.STRING) String reason,
@Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
@@ -360,9 +416,9 @@ public class Section {
EntitySearchUtils.addEntitiesIgnoreRank(entities, found);
}
+
@ThenAction
- public void redactLineAfter(@Argument(ArgumentType.STRING) String start,
- @Argument(ArgumentType.TYPE) String asType,
+ public void redactLineAfter(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.TYPE) String asType,
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
@Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
@Argument(ArgumentType.STRING) String reason,
@@ -384,6 +440,7 @@ public class Section {
}
}
+
@ThenAction
public void recommendLineAfter(@Argument(ArgumentType.STRING) String start,
@Argument(ArgumentType.TYPE) String asType) {
@@ -409,11 +466,11 @@ public class Section {
}
}
+
@ThenAction
public void redactByRegEx(@Argument(ArgumentType.REGEX) String pattern,
@Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
- @Argument(ArgumentType.INTEGER) int group,
- @Argument(ArgumentType.TYPE) String asType,
+ @Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.TYPE) String asType,
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
@Argument(ArgumentType.STRING) String reason,
@Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
@@ -431,6 +488,7 @@ public class Section {
}
}
+
@ThenAction
public void addRecommendationByRegEx(@Argument(ArgumentType.REGEX) String pattern,
@Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
@@ -449,6 +507,7 @@ public class Section {
}
}
+
@ThenAction
public void redactAndRecommendByRegEx(@Argument(ArgumentType.REGEX) String pattern,
@Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
@@ -471,9 +530,9 @@ public class Section {
}
}
+
@ThenAction
- public void redactBetween(@Argument(ArgumentType.STRING) String start,
- @Argument(ArgumentType.STRING) String stop,
+ public void redactBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop,
@Argument(ArgumentType.TYPE) String asType,
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
@Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
@@ -497,6 +556,7 @@ public class Section {
}
}
+
@ThenAction
public void redactLinesBetween(@Argument(ArgumentType.STRING) String start,
@Argument(ArgumentType.STRING) String stop,
@@ -531,6 +591,7 @@ public class Section {
}
}
+
@ThenAction
public void highlightCell(@Argument(ArgumentType.STRING) String cellHeader,
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
@@ -539,10 +600,10 @@ public class Section {
annotateCell(cellHeader, ruleNumber, type, false, false, null, null);
}
+
@ThenAction
public void redactCell(@Argument(ArgumentType.STRING) String cellHeader,
- @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
- @Argument(ArgumentType.TYPE) String type,
+ @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.TYPE) String type,
@Argument(ArgumentType.BOOLEAN) boolean addAsRecommendations,
@Argument(ArgumentType.STRING) String reason,
@Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
@@ -550,6 +611,7 @@ public class Section {
annotateCell(cellHeader, ruleNumber, type, true, addAsRecommendations, reason, legalBasis);
}
+
@ThenAction
public void redactNotCell(@Argument(ArgumentType.STRING) String cellHeader,
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
@@ -636,6 +698,7 @@ public class Section {
}
}
+
@Retention(RetentionPolicy.RUNTIME)
@Target(ElementType.METHOD)
public @interface WhenCondition {
@@ -653,6 +716,7 @@ public class Section {
public @interface Argument {
ArgumentType value() default ArgumentType.STRING;
+
}
}
diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java
index 3a2282fb..167b30f2 100644
--- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java
+++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java
@@ -1,17 +1,36 @@
package com.iqser.red.service.redaction.v1.server.redaction.service;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.*;
+import java.time.OffsetDateTime;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.IdRemoval;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualForceRedaction;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualImageRecategorization;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualLegalBasisChange;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactionEntry;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualResizeRedaction;
+import com.iqser.red.service.redaction.v1.model.ManualRedactionType;
+import com.iqser.red.service.redaction.v1.model.Point;
import com.iqser.red.service.redaction.v1.model.Rectangle;
-import com.iqser.red.service.redaction.v1.model.*;
+import com.iqser.red.service.redaction.v1.model.RedactionLog;
+import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
+import com.iqser.red.service.redaction.v1.model.SectionGrid;
+
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
-import org.springframework.stereotype.Service;
-
-import java.time.OffsetDateTime;
-import java.util.*;
-import java.util.stream.Collectors;
@Slf4j
@Service
@@ -23,8 +42,9 @@ public class RedactionLogMergeService {
private final SectionTextService sectionTextService;
- public RedactionLog mergeRedactionLogData(RedactionLog redactionLog, SectionGrid sectionGrid, String dossierTemplateId,
- ManualRedactions manualRedactions, Set excludedPages) {
+ public RedactionLog mergeRedactionLogData(RedactionLog redactionLog, SectionGrid sectionGrid,
+ String dossierTemplateId, ManualRedactions manualRedactions,
+ Set excludedPages) {
log.info("Merging Redaction log with manual redactions");
if (manualRedactions != null) {
@@ -54,6 +74,15 @@ public class RedactionLogMergeService {
}
+ Set processedIds = new HashSet<>();
+ redactionLog.getRedactionLogEntry().removeIf(entry -> {
+ if (processedIds.contains(entry.getId())) {
+ log.info("Duplicate annotation found with id {}", entry.getId());
+ return true;
+ }
+ processedIds.add(entry.getId());
+ return false;
+ });
return redactionLog;
}
@@ -111,12 +140,19 @@ public class RedactionLogMergeService {
redactionLogEntry.setType(imageRecategorization.getType());
redactionLogEntry.setHasBeenRecategorized(true);
redactionLogEntry.setSection("Image:" + redactionLogEntry.getType());
+
+ if (dictionaryService.isHint(imageRecategorization.getType(), dossierTemplateId)) {
+ redactionLogEntry.setRedacted(false);
+ redactionLogEntry.setHint(true);
+ } else {
+ redactionLogEntry.setHint(false);
+ }
+
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", recategorized by manual override");
} else if (imageRecategorization.getStatus().equals(AnnotationStatus.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to recategorize");
redactionLogEntry.setStatus(AnnotationStatus.REQUESTED);
- redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry
- .isRedacted(), false));
+ redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted(), false));
redactionLogEntry.setRecategorizationType(imageRecategorization.getType());
} else {
redactionLogEntry.setStatus(AnnotationStatus.DECLINED);
@@ -135,14 +171,12 @@ public class RedactionLogMergeService {
redactionLogEntry.setRedacted(false);
redactionLogEntry.setStatus(AnnotationStatus.APPROVED);
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", removed by manual override");
- redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry
- .isRedacted(), true));
+ redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted(), true));
redactionLogEntry.setHasBeenRemovedByManualOverride(true);
} else if (manualRemoval.getStatus().equals(AnnotationStatus.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to remove");
redactionLogEntry.setStatus(AnnotationStatus.REQUESTED);
- redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry
- .isRedacted(), false));
+ redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted(), false));
} else {
redactionLogEntry.setStatus(AnnotationStatus.DECLINED);
}
@@ -161,16 +195,14 @@ public class RedactionLogMergeService {
if (manualForceRedact.getStatus().equals(AnnotationStatus.APPROVED)) {
redactionLogEntry.setRedacted(true);
redactionLogEntry.setStatus(AnnotationStatus.APPROVED);
- redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry
- .isRedacted(), false));
+ redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted(), false));
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", forced by manual override");
redactionLogEntry.setHasBeenForced(true);
redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
} else if (manualForceRedact.getStatus().equals(AnnotationStatus.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to force redact");
redactionLogEntry.setStatus(AnnotationStatus.REQUESTED);
- redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry
- .isRedacted(), false));
+ redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted(), false));
redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
} else {
redactionLogEntry.setStatus(AnnotationStatus.DECLINED);
@@ -195,11 +227,13 @@ public class RedactionLogMergeService {
if (manualLegalBasisChange.getSection() != null) {
redactionLogEntry.setSection(manualLegalBasisChange.getSection());
}
+ if (redactionLogEntry.isRectangle() && manualLegalBasisChange.getValue() != null) {
+ redactionLogEntry.setValue(manualLegalBasisChange.getValue());
+ }
} else if (manualLegalBasisChange.getStatus().equals(AnnotationStatus.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis change requested");
redactionLogEntry.setStatus(AnnotationStatus.REQUESTED);
- redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry
- .isRedacted(), false));
+ redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted(), false));
redactionLogEntry.setLegalBasisChangeValue(manualLegalBasisChange.getLegalBasis());
} else {
redactionLogEntry.setStatus(AnnotationStatus.DECLINED);
@@ -216,8 +250,7 @@ public class RedactionLogMergeService {
String manualOverrideReason = null;
if (manualResizeRedact.getStatus().equals(AnnotationStatus.APPROVED)) {
redactionLogEntry.setStatus(AnnotationStatus.APPROVED);
- redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry
- .isRedacted(), false));
+ redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted(), false));
redactionLogEntry.setPositions(convertPositions(manualResizeRedact.getPositions()));
redactionLogEntry.setValue(manualResizeRedact.getValue());
redactionLogEntry.setHasBeenResized(true);
@@ -225,8 +258,7 @@ public class RedactionLogMergeService {
} else if (manualResizeRedact.getStatus().equals(AnnotationStatus.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to resize redact");
redactionLogEntry.setStatus(AnnotationStatus.REQUESTED);
- redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry
- .isRedacted(), false));
+ redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted(), false));
redactionLogEntry.setPositions(convertPositions(manualResizeRedact.getPositions()));
} else {
redactionLogEntry.setStatus(AnnotationStatus.DECLINED);
@@ -278,9 +310,13 @@ public class RedactionLogMergeService {
return redactionLogEntries;
}
- private List convertPositions(List positions) {
- return positions.stream().map(pos -> new Rectangle(new Point(pos.getTopLeftX(), pos.getTopLeftY()), pos.getWidth(), pos
- .getHeight(), pos.getPage())).collect(Collectors.toList());
+
+ private List convertPositions(
+ List positions) {
+
+ return positions.stream()
+ .map(pos -> new Rectangle(new Point(pos.getTopLeftX(), pos.getTopLeftY()), pos.getWidth(), pos.getHeight(), pos.getPage()))
+ .collect(Collectors.toList());
}
@@ -311,6 +347,7 @@ public class RedactionLogMergeService {
.status(manualRedactionEntry.getStatus())
.manualRedactionType(ManualRedactionType.ADD)
.manualRedactionUserId(manualRedactionEntry.getUser())
+ .rectangle(manualRedactionEntry.isRectangle())
.build();
}
@@ -362,6 +399,5 @@ public class RedactionLogMergeService {
}
-
}
diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java
index d514df1e..7d101356 100644
--- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java
+++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java
@@ -1,5 +1,17 @@
package com.iqser.red.service.redaction.v1.server.redaction.utils;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
import com.iqser.red.service.redaction.v1.model.Engine;
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrementValue;
@@ -10,10 +22,6 @@ import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
import lombok.experimental.UtilityClass;
import lombok.extern.slf4j.Slf4j;
-import java.util.*;
-import java.util.regex.Pattern;
-import java.util.stream.Collectors;
-
@Slf4j
@UtilityClass
@SuppressWarnings("PMD")
@@ -37,8 +45,7 @@ public class EntitySearchUtils {
startIndex = inputString.indexOf(cleanValue, stopIndex);
stopIndex = startIndex + cleanValue.length();
- if (startIndex > -1 && (startIndex == 0 || Character.isWhitespace(inputString.charAt(startIndex - 1)) || isSeparator(inputString
- .charAt(startIndex - 1))) && (stopIndex == inputString.length() || isSeparator(inputString.charAt(stopIndex)))) {
+ if (startIndex > -1 && (startIndex == 0 || Character.isWhitespace(inputString.charAt(startIndex - 1)) || isSeparator(inputString.charAt(startIndex - 1))) && (stopIndex == inputString.length() || isSeparator(inputString.charAt(stopIndex)))) {
return true;
}
} while (startIndex > -1);
@@ -66,8 +73,7 @@ public class EntitySearchUtils {
startIndex = inputString.indexOf(cleanValue, stopIndex);
stopIndex = startIndex + cleanValue.length();
- if (startIndex > -1 && (startIndex == 0 || Character.isWhitespace(inputString.charAt(startIndex - 1)) || isSeparator(inputString
- .charAt(startIndex - 1))) && (stopIndex == inputString.length() || isSeparator(inputString.charAt(stopIndex)))) {
+ if (startIndex > -1 && (startIndex == 0 || Character.isWhitespace(inputString.charAt(startIndex - 1)) || isSeparator(inputString.charAt(startIndex - 1))) && (stopIndex == inputString.length() || isSeparator(inputString.charAt(stopIndex)))) {
found.add(new Entity(inputString.substring(startIndex, stopIndex), type, startIndex, stopIndex, headline, sectionNumber, isDictionaryEntry, isDossierDictionary, engine));
}
} while (startIndex > -1);
@@ -121,8 +127,7 @@ public class EntitySearchUtils {
for (Entity word : entities) {
for (Entity inner : entities) {
if (inner.getWord().length() < word.getWord()
- .length() && inner.getStart() >= word.getStart() && inner.getEnd() <= word.getEnd() && word != inner && word
- .getSectionNumber() == inner.getSectionNumber()) {
+ .length() && inner.getStart() >= word.getStart() && inner.getEnd() <= word.getEnd() && word != inner && word.getSectionNumber() == inner.getSectionNumber()) {
wordsToRemove.add(inner);
}
}
@@ -141,14 +146,14 @@ public class EntitySearchUtils {
if (entities.contains(found)) {
Optional existingOptional = entities.stream().filter(entity -> entity.equals(found)).findFirst();
- if(!existingOptional.isPresent()){
+ if (!existingOptional.isPresent()) {
return;
}
var existing = existingOptional.get();
- if(existing.getType().equals(found.getType())){
+ if (existing.getType().equals(found.getType())) {
existing.getEngines().addAll(found.getEngines());
- } else if (dictionary.getDictionaryRank(existing.getType()) <= dictionary.getDictionaryRank(found.getType())){
+ } else if (dictionary.getDictionaryRank(existing.getType()) <= dictionary.getDictionaryRank(found.getType())) {
entities.remove(found);
entities.add(found);
}
@@ -165,12 +170,14 @@ public class EntitySearchUtils {
}
- public void addOrAddEngine(Set existing, Set toBeAdded){
+ public void addOrAddEngine(Set existing, Set toBeAdded) {
- for(Entity toAdd: toBeAdded){
+ for (Entity toAdd : toBeAdded) {
if (existing.contains(toAdd)) {
- Optional existingOptional = existing.stream().filter(entity -> entity.equals(toAdd)).findFirst();
- if(!existingOptional.isPresent()){
+ Optional existingOptional = existing.stream()
+ .filter(entity -> entity.equals(toAdd))
+ .findFirst();
+ if (!existingOptional.isPresent()) {
return;
}
var existingEntity = existingOptional.get();
@@ -181,4 +188,20 @@ public class EntitySearchUtils {
}
}
+
+ public Set findNonOverlappingMatchEntities(Set existingEntities, Set foundEntities) {
+
+ Set result = new HashSet<>();
+ if (existingEntities != null && foundEntities != null) {
+ for (Entity existingEntity : existingEntities) {
+ for (Entity foundEntity : foundEntities) {
+ if (existingEntity.getEnd() < foundEntity.getStart() || foundEntity.getEnd() < existingEntity.getStart()) {
+ result.add(foundEntity);
+ }
+ }
+ }
+ }
+ return result;
+ }
+
}
diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtilsTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtilsTest.java
index ae2fb019..eec7d419 100644
--- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtilsTest.java
+++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtilsTest.java
@@ -27,4 +27,164 @@ public class EntitySearchUtilsTest {
}
+
+ /*
+ * Text: Batman X. Superman Y.
+ * Position: 0123456789
+ * 0123456789
+ * 0123456789
+ */
+ @Test
+ public void testNotOverlappingEntitiesExpandedEnd() {
+
+ // Arrange
+ Set existingEntities = new HashSet<>();
+ Entity existingEntity1 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, false, false, Engine.RULE);
+ Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE);
+ existingEntities.add(existingEntity1);
+ existingEntities.add(existingEntity2);
+
+ Set foundEntities = new HashSet<>();
+ Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE);
+ Entity foundEntities2 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, false, false, Engine.RULE);
+ foundEntities.add(foundEntities1);
+ foundEntities.add(foundEntities2);
+
+ // Act
+ Set result = EntitySearchUtils.findNonOverlappingMatchEntities(existingEntities, foundEntities);
+
+ // Assert
+ assertThat(result.size()).isEqualTo(2);
+ assertThat(result).contains(foundEntities1);
+ assertThat(result).contains(foundEntities2);
+
+ }
+
+
+ /*
+ * Text: Batman X. Superman Y.
+ * Position: 0123456789
+ * 0123456789
+ * 0123456789
+ */
+ @Test
+ public void testNotOverlappingEntitiesExpandedStartAndEndOverlapping() {
+
+ // Arrange
+ Set existingEntities = new HashSet<>();
+ Entity existingEntity1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE);
+ Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE);
+ existingEntities.add(existingEntity1);
+ existingEntities.add(existingEntity2);
+
+ Set foundEntities = new HashSet<>();
+ Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE);
+ Entity foundEntities2 = new Entity("X. Superman Y.", "fake type", 7, 20, "fake headline", 0, false, false, Engine.RULE);
+ foundEntities.add(foundEntities1);
+ foundEntities.add(foundEntities2);
+
+ // Act
+ Set result = EntitySearchUtils.findNonOverlappingMatchEntities(existingEntities, foundEntities);
+
+ // Assert
+ assertThat(result.size()).isEqualTo(1);
+ assertThat(result).contains(foundEntities1);
+
+ }
+
+
+ /*
+ * Text: Batman X. Superman Y.
+ * Position: 0123456789
+ * 0123456789
+ * 0123456789
+ */
+ @Test
+ public void testNotOverlappingEntitiesExpandedStartAndEnd() {
+
+ // Arrange
+ Set existingEntities = new HashSet<>();
+ Entity existingEntity1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE);
+ Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE);
+ existingEntities.add(existingEntity1);
+ existingEntities.add(existingEntity2);
+
+ Set foundEntities = new HashSet<>();
+ Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE);
+ Entity foundEntities2 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, false, false, Engine.RULE);
+ foundEntities.add(foundEntities1);
+ foundEntities.add(foundEntities2);
+
+ // Act
+ Set result = EntitySearchUtils.findNonOverlappingMatchEntities(existingEntities, foundEntities);
+
+ // Assert
+ assertThat(result.size()).isEqualTo(1);
+ assertThat(result).contains(foundEntities1);
+
+ }
+
+
+ /*
+ * Text: Batman X. Superman Y.
+ * Position: 0123456789
+ * 0123456789
+ * 0123456789
+ */
+ @Test
+ public void testNotOverlappingEntitiesExpandedExistingAndExpandedEnd() {
+
+ // Arrange
+ Set existingEntities = new HashSet<>();
+ Entity existingEntity1 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, false, false, Engine.RULE);
+ Entity existingEntity2 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, false, false, Engine.RULE);
+ existingEntities.add(existingEntity1);
+ existingEntities.add(existingEntity2);
+
+ Set foundEntities = new HashSet<>();
+ Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE);
+ Entity foundEntities2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE);
+ foundEntities.add(foundEntities1);
+ foundEntities.add(foundEntities2);
+
+ // Act
+ Set result = EntitySearchUtils.findNonOverlappingMatchEntities(existingEntities, foundEntities);
+
+ // Assert
+ assertThat(result.size()).isEqualTo(1);
+ assertThat(result).contains(foundEntities2);
+
+ }
+
+ /*
+ * Text: Batman X. Superman Y.
+ * Position: 0123456789
+ * 0123456789
+ * 0123456789
+ */
+ @Test
+ public void testNotOverlappingEntitiesExpandedEndLong() {
+
+ // Arrange
+ Set existingEntities = new HashSet<>();
+ Entity existingEntity1 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, false, false, Engine.RULE);
+ Entity existingEntity2 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, false, false, Engine.RULE);
+ existingEntities.add(existingEntity1);
+ existingEntities.add(existingEntity2);
+
+ Set foundEntities = new HashSet<>();
+ Entity foundEntities1 = new Entity("Batman X. Superman", "fake type", 0, 17, "fake headline", 0, false, false, Engine.RULE);
+ Entity foundEntities2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE);
+ foundEntities.add(foundEntities1);
+ foundEntities.add(foundEntities2);
+
+ // Act
+ Set result = EntitySearchUtils.findNonOverlappingMatchEntities(existingEntities, foundEntities);
+
+ // Assert
+ assertThat(result.size()).isEqualTo(1);
+ assertThat(result).contains(foundEntities2);
+
+ }
+
}
diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/CBI_author.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/CBI_author.txt
index e2854b4c..9cf34080 100644
--- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/CBI_author.txt
+++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/CBI_author.txt
@@ -1,3 +1,6 @@
+Foo
+F. Bar
+B. Foo
Johnson R |
Weissler M S and Butters C A
AD Hurt
diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl
index c0b96ff9..007fea23 100644
--- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl
+++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl
@@ -19,8 +19,8 @@ rule "0: Expand CBI Authors with firstname initials"
when
Section(matchesType("CBI_author") || matchesType("recommendation_CBI_author"))
then
- section.expandByRegEx("CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1, "[^\\s]+");
- section.expandByRegEx("recommendation_CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1, "[^\\s]+");
+ section.expandByRegEx("CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1, "[\\s]+");
+ section.expandByRegEx("recommendation_CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1, "[\\s]+");
end
@@ -344,4 +344,11 @@ rule "29: Redact Dossier Redactions"
Section(matchesType("dossier_redactions"))
then
section.redact("dossier_redactions", 29, "Dossier Redaction found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
- end
\ No newline at end of file
+ end
+
+rule "30: Ignore dossier_redactions if confidential"
+ when
+ Section(!fileAttributeByLabelEqualsIgnoreCase("Confidentiality","confidential") && matchesType("dossier_redactions"));
+ then
+ section.ignore("dossier_redactions");
+ end
\ No newline at end of file
diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/Minimal Examples/foo.pdf b/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/Minimal Examples/foo.pdf
new file mode 100644
index 00000000..b6ddcbc0
Binary files /dev/null and b/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/Minimal Examples/foo.pdf differ