diff --git a/redaction-service-v1/redaction-service-api-v1/pom.xml b/redaction-service-v1/redaction-service-api-v1/pom.xml index 5337144b..fa22253a 100644 --- a/redaction-service-v1/redaction-service-api-v1/pom.xml +++ b/redaction-service-v1/redaction-service-api-v1/pom.xml @@ -12,7 +12,7 @@ redaction-service-api-v1 - 0.103.0 + 0.121.0 diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java index 8967510d..b23d7b59 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java @@ -26,6 +26,7 @@ public class RedactionLogEntry { private String value; private String reason; private int matchedRule; + private boolean rectangle; private String legalBasis; private boolean redacted; private boolean isHint; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java index ac1b0dde..49b8f310 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java @@ -1,26 +1,34 @@ package com.iqser.red.service.redaction.v1.server.redaction.model; +import static com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary.RECOMMENDATION_PREFIX; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +import org.apache.commons.lang3.StringUtils; + import com.iqser.red.service.redaction.v1.model.ArgumentType; import com.iqser.red.service.redaction.v1.model.Engine; import com.iqser.red.service.redaction.v1.model.FileAttribute; import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock; import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils; import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns; + import lombok.Builder; import lombok.Data; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; - -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; -import java.util.*; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import java.util.stream.Collectors; - -import static com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary.RECOMMENDATION_PREFIX; @Data @Slf4j @@ -62,39 +70,69 @@ public class Section { @WhenCondition public boolean fileAttributeByIdEquals(@Argument(ArgumentType.FILE_ATTRIBUTE) String id, @Argument(ArgumentType.STRING) String value) { - return fileAttributes != null && fileAttributes.stream().filter(attribute -> id.equals(attribute.getId()) && value.equals(attribute.getValue())).findFirst().isPresent(); + + return fileAttributes != null && fileAttributes.stream() + .filter(attribute -> id.equals(attribute.getId()) && value.equals(attribute.getValue())) + .findFirst() + .isPresent(); } + @WhenCondition public boolean fileAttributeByPlaceholderEquals(@Argument(ArgumentType.FILE_ATTRIBUTE) String placeholder, @Argument(ArgumentType.STRING) String value) { - return fileAttributes != null && fileAttributes.stream().filter(attribute -> placeholder.equals(attribute.getPlaceholder()) && value.equals(attribute.getValue())).findFirst().isPresent(); + + return fileAttributes != null && fileAttributes.stream() + .filter(attribute -> placeholder.equals(attribute.getPlaceholder()) && value.equals(attribute.getValue())) + .findFirst() + .isPresent(); } + @WhenCondition public boolean fileAttributeByLabelEquals(@Argument(ArgumentType.FILE_ATTRIBUTE) String label, @Argument(ArgumentType.STRING) String value) { - return fileAttributes != null && fileAttributes.stream().filter(attribute -> label.equals(attribute.getLabel()) && value.equals(attribute.getValue())).findFirst().isPresent(); + + return fileAttributes != null && fileAttributes.stream() + .filter(attribute -> label.equals(attribute.getLabel()) && value.equals(attribute.getValue())) + .findFirst() + .isPresent(); } + @WhenCondition public boolean fileAttributeByIdEqualsIgnoreCase(@Argument(ArgumentType.FILE_ATTRIBUTE) String id, @Argument(ArgumentType.STRING) String value) { - return fileAttributes != null && fileAttributes.stream().filter(attribute -> id.equals(attribute.getId()) && value.equalsIgnoreCase(attribute.getValue())).findFirst().isPresent(); + + return fileAttributes != null && fileAttributes.stream() + .filter(attribute -> id.equals(attribute.getId()) && value.equalsIgnoreCase(attribute.getValue())) + .findFirst() + .isPresent(); } + @WhenCondition public boolean fileAttributeByPlaceholderEqualsIgnoreCase(@Argument(ArgumentType.FILE_ATTRIBUTE) String placeholder, @Argument(ArgumentType.STRING) String value) { - return fileAttributes != null && fileAttributes.stream().filter(attribute -> placeholder.equals(attribute.getPlaceholder()) && value.equalsIgnoreCase(attribute.getValue())).findFirst().isPresent(); + + return fileAttributes != null && fileAttributes.stream() + .filter(attribute -> placeholder.equals(attribute.getPlaceholder()) && value.equalsIgnoreCase(attribute.getValue())) + .findFirst() + .isPresent(); } + @WhenCondition public boolean fileAttributeByLabelEqualsIgnoreCase(@Argument(ArgumentType.FILE_ATTRIBUTE) String label, @Argument(ArgumentType.STRING) String value) { - return fileAttributes != null && fileAttributes.stream().filter(attribute -> label.equals(attribute.getLabel()) && value.equalsIgnoreCase(attribute.getValue())).findFirst().isPresent(); + + return fileAttributes != null && fileAttributes.stream() + .filter(attribute -> label.equals(attribute.getLabel()) && value.equalsIgnoreCase(attribute.getValue())) + .findFirst() + .isPresent(); } + @WhenCondition public boolean rowEquals(@Argument(ArgumentType.STRING) String headerName, @Argument(ArgumentType.STRING) String value) { @@ -106,6 +144,7 @@ public class Section { .equals(value); } + @WhenCondition public boolean hasTableHeader(@Argument(ArgumentType.STRING) String headerName) { @@ -113,18 +152,22 @@ public class Section { return tabularData != null && tabularData.containsKey(cleanHeaderName); } + @WhenCondition public boolean matchesType(@Argument(ArgumentType.TYPE) String type) { return entities.stream().anyMatch(entity -> entity.getType().equals(type)); } + @WhenCondition public boolean matchesImageType(@Argument(ArgumentType.TYPE) String type) { + return images.stream().anyMatch(image -> image.getType().equals(type)); } + @WhenCondition public boolean headlineContainsWord(@Argument(ArgumentType.STRING) String word) { @@ -133,13 +176,14 @@ public class Section { @ThenAction - public void expandByRegEx(@Argument(ArgumentType.TYPE) String type, - @Argument(ArgumentType.REGEX) String pattern, + public void expandByRegEx(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.REGEX) String pattern, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, @Argument(ArgumentType.INTEGER) int group) { + expandByRegEx(type, pattern, patternCaseInsensitive, group, null); } + @ThenAction public void expandByRegEx(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.REGEX) String suffixPattern, @@ -162,7 +206,7 @@ public class Section { continue; } - if(compiledValuePattern != null) { + if (compiledValuePattern != null) { Matcher matcherCompiledValuePattern = compiledValuePat.matcher(entity.getWord()); if (!matcherCompiledValuePattern.find()) { continue; @@ -173,10 +217,10 @@ public class Section { while (matcher.find()) { String match = matcher.group(group); - if (StringUtils.isNotBlank(match)) { - expanded.addAll(findEntities(entity.getWord() + match, type, false, entity.isRedaction(), entity.getMatchedRule(), entity - .getRedactionReason(), entity.getLegalBasis())); + if (StringUtils.isNotBlank(match)) { + Set expandedEntities = findEntities(entity.getWord() + match, type, false, entity.isRedaction(), entity.getMatchedRule(), entity.getRedactionReason(), entity.getLegalBasis()); + expanded.addAll(EntitySearchUtils.findNonOverlappingMatchEntities(entities, expandedEntities)); } } } @@ -185,6 +229,7 @@ public class Section { EntitySearchUtils.removeEntitiesContainedInLarger(entities); } + @ThenAction public void redactImage(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @@ -201,9 +246,9 @@ public class Section { }); } + @ThenAction - public void redact(@Argument(ArgumentType.TYPE) String type, - @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, + public void redact(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { @@ -220,6 +265,7 @@ public class Section { }); } + @ThenAction public void redactNotImage(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @@ -234,9 +280,9 @@ public class Section { }); } + @ThenAction - public void redactNot(@Argument(ArgumentType.TYPE) String type, - @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, + public void redactNot(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason) { boolean hasRecommendationDictionary = dictionaryTypes.contains(RECOMMENDATION_PREFIX + type); @@ -252,15 +298,23 @@ public class Section { } + public void ignore(String type) { + + entities.removeIf(entity -> entity.getType().equals(type)); + } + + @ThenAction public void redactNotAndReference(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.REFERENCE_TYPE) String referenceType, - @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, - @Argument(ArgumentType.STRING) String reason) { + @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, + @Argument(ArgumentType.STRING) String reason) { boolean hasRecommendationDictionary = dictionaryTypes.contains(RECOMMENDATION_PREFIX + type); - Set references = entities.stream().filter(entity -> entity.getType().equals(referenceType)).collect(Collectors.toSet()); + Set references = entities.stream() + .filter(entity -> entity.getType().equals(referenceType)) + .collect(Collectors.toSet()); entities.forEach(entity -> { if (entity.getType().equals(type) || hasRecommendationDictionary && entity.getType() @@ -274,7 +328,6 @@ public class Section { } - @ThenAction public void expandToHintAnnotationByRegEx(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.STRING) String pattern, @@ -305,6 +358,7 @@ public class Section { EntitySearchUtils.removeEntitiesContainedInLarger(entities); } + @ThenAction public void addHintAnnotationByRegEx(@Argument(ArgumentType.REGEX) String pattern, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, @@ -324,6 +378,7 @@ public class Section { } } + @ThenAction public void redactIfPrecededBy(@Argument(ArgumentType.STRING) String prefix, @Argument(ArgumentType.TYPE) String type, @@ -341,6 +396,7 @@ public class Section { }); } + @ThenAction public void addHintAnnotation(@Argument(ArgumentType.STRING) String value, @Argument(ArgumentType.TYPE) String asType) { @@ -349,9 +405,9 @@ public class Section { EntitySearchUtils.addEntitiesIgnoreRank(entities, found); } + @ThenAction - public void addRedaction(@Argument(ArgumentType.STRING) String value, - @Argument(ArgumentType.TYPE) String asType, + public void addRedaction(@Argument(ArgumentType.STRING) String value, @Argument(ArgumentType.TYPE) String asType, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { @@ -360,9 +416,9 @@ public class Section { EntitySearchUtils.addEntitiesIgnoreRank(entities, found); } + @ThenAction - public void redactLineAfter(@Argument(ArgumentType.STRING) String start, - @Argument(ArgumentType.TYPE) String asType, + public void redactLineAfter(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.TYPE) String asType, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere, @Argument(ArgumentType.STRING) String reason, @@ -384,6 +440,7 @@ public class Section { } } + @ThenAction public void recommendLineAfter(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.TYPE) String asType) { @@ -409,11 +466,11 @@ public class Section { } } + @ThenAction public void redactByRegEx(@Argument(ArgumentType.REGEX) String pattern, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, - @Argument(ArgumentType.INTEGER) int group, - @Argument(ArgumentType.TYPE) String asType, + @Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.TYPE) String asType, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { @@ -431,6 +488,7 @@ public class Section { } } + @ThenAction public void addRecommendationByRegEx(@Argument(ArgumentType.REGEX) String pattern, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, @@ -449,6 +507,7 @@ public class Section { } } + @ThenAction public void redactAndRecommendByRegEx(@Argument(ArgumentType.REGEX) String pattern, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, @@ -471,9 +530,9 @@ public class Section { } } + @ThenAction - public void redactBetween(@Argument(ArgumentType.STRING) String start, - @Argument(ArgumentType.STRING) String stop, + public void redactBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere, @@ -497,6 +556,7 @@ public class Section { } } + @ThenAction public void redactLinesBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop, @@ -531,6 +591,7 @@ public class Section { } } + @ThenAction public void highlightCell(@Argument(ArgumentType.STRING) String cellHeader, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @@ -539,10 +600,10 @@ public class Section { annotateCell(cellHeader, ruleNumber, type, false, false, null, null); } + @ThenAction public void redactCell(@Argument(ArgumentType.STRING) String cellHeader, - @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, - @Argument(ArgumentType.TYPE) String type, + @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.BOOLEAN) boolean addAsRecommendations, @Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { @@ -550,6 +611,7 @@ public class Section { annotateCell(cellHeader, ruleNumber, type, true, addAsRecommendations, reason, legalBasis); } + @ThenAction public void redactNotCell(@Argument(ArgumentType.STRING) String cellHeader, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @@ -636,6 +698,7 @@ public class Section { } } + @Retention(RetentionPolicy.RUNTIME) @Target(ElementType.METHOD) public @interface WhenCondition { @@ -653,6 +716,7 @@ public class Section { public @interface Argument { ArgumentType value() default ArgumentType.STRING; + } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java index 3a2282fb..167b30f2 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java @@ -1,17 +1,36 @@ package com.iqser.red.service.redaction.v1.server.redaction.service; -import com.iqser.red.service.persistence.service.v1.api.model.annotations.*; +import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import org.springframework.stereotype.Service; + +import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.IdRemoval; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualForceRedaction; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualImageRecategorization; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualLegalBasisChange; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactionEntry; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualResizeRedaction; +import com.iqser.red.service.redaction.v1.model.ManualRedactionType; +import com.iqser.red.service.redaction.v1.model.Point; import com.iqser.red.service.redaction.v1.model.Rectangle; -import com.iqser.red.service.redaction.v1.model.*; +import com.iqser.red.service.redaction.v1.model.RedactionLog; +import com.iqser.red.service.redaction.v1.model.RedactionLogEntry; +import com.iqser.red.service.redaction.v1.model.SectionGrid; + import lombok.AllArgsConstructor; import lombok.Data; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Service; - -import java.time.OffsetDateTime; -import java.util.*; -import java.util.stream.Collectors; @Slf4j @Service @@ -23,8 +42,9 @@ public class RedactionLogMergeService { private final SectionTextService sectionTextService; - public RedactionLog mergeRedactionLogData(RedactionLog redactionLog, SectionGrid sectionGrid, String dossierTemplateId, - ManualRedactions manualRedactions, Set excludedPages) { + public RedactionLog mergeRedactionLogData(RedactionLog redactionLog, SectionGrid sectionGrid, + String dossierTemplateId, ManualRedactions manualRedactions, + Set excludedPages) { log.info("Merging Redaction log with manual redactions"); if (manualRedactions != null) { @@ -54,6 +74,15 @@ public class RedactionLogMergeService { } + Set processedIds = new HashSet<>(); + redactionLog.getRedactionLogEntry().removeIf(entry -> { + if (processedIds.contains(entry.getId())) { + log.info("Duplicate annotation found with id {}", entry.getId()); + return true; + } + processedIds.add(entry.getId()); + return false; + }); return redactionLog; } @@ -111,12 +140,19 @@ public class RedactionLogMergeService { redactionLogEntry.setType(imageRecategorization.getType()); redactionLogEntry.setHasBeenRecategorized(true); redactionLogEntry.setSection("Image:" + redactionLogEntry.getType()); + + if (dictionaryService.isHint(imageRecategorization.getType(), dossierTemplateId)) { + redactionLogEntry.setRedacted(false); + redactionLogEntry.setHint(true); + } else { + redactionLogEntry.setHint(false); + } + manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", recategorized by manual override"); } else if (imageRecategorization.getStatus().equals(AnnotationStatus.REQUESTED)) { manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to recategorize"); redactionLogEntry.setStatus(AnnotationStatus.REQUESTED); - redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry - .isRedacted(), false)); + redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted(), false)); redactionLogEntry.setRecategorizationType(imageRecategorization.getType()); } else { redactionLogEntry.setStatus(AnnotationStatus.DECLINED); @@ -135,14 +171,12 @@ public class RedactionLogMergeService { redactionLogEntry.setRedacted(false); redactionLogEntry.setStatus(AnnotationStatus.APPROVED); manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", removed by manual override"); - redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry - .isRedacted(), true)); + redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted(), true)); redactionLogEntry.setHasBeenRemovedByManualOverride(true); } else if (manualRemoval.getStatus().equals(AnnotationStatus.REQUESTED)) { manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to remove"); redactionLogEntry.setStatus(AnnotationStatus.REQUESTED); - redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry - .isRedacted(), false)); + redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted(), false)); } else { redactionLogEntry.setStatus(AnnotationStatus.DECLINED); } @@ -161,16 +195,14 @@ public class RedactionLogMergeService { if (manualForceRedact.getStatus().equals(AnnotationStatus.APPROVED)) { redactionLogEntry.setRedacted(true); redactionLogEntry.setStatus(AnnotationStatus.APPROVED); - redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry - .isRedacted(), false)); + redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted(), false)); manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", forced by manual override"); redactionLogEntry.setHasBeenForced(true); redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis()); } else if (manualForceRedact.getStatus().equals(AnnotationStatus.REQUESTED)) { manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to force redact"); redactionLogEntry.setStatus(AnnotationStatus.REQUESTED); - redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry - .isRedacted(), false)); + redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted(), false)); redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis()); } else { redactionLogEntry.setStatus(AnnotationStatus.DECLINED); @@ -195,11 +227,13 @@ public class RedactionLogMergeService { if (manualLegalBasisChange.getSection() != null) { redactionLogEntry.setSection(manualLegalBasisChange.getSection()); } + if (redactionLogEntry.isRectangle() && manualLegalBasisChange.getValue() != null) { + redactionLogEntry.setValue(manualLegalBasisChange.getValue()); + } } else if (manualLegalBasisChange.getStatus().equals(AnnotationStatus.REQUESTED)) { manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis change requested"); redactionLogEntry.setStatus(AnnotationStatus.REQUESTED); - redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry - .isRedacted(), false)); + redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted(), false)); redactionLogEntry.setLegalBasisChangeValue(manualLegalBasisChange.getLegalBasis()); } else { redactionLogEntry.setStatus(AnnotationStatus.DECLINED); @@ -216,8 +250,7 @@ public class RedactionLogMergeService { String manualOverrideReason = null; if (manualResizeRedact.getStatus().equals(AnnotationStatus.APPROVED)) { redactionLogEntry.setStatus(AnnotationStatus.APPROVED); - redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry - .isRedacted(), false)); + redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted(), false)); redactionLogEntry.setPositions(convertPositions(manualResizeRedact.getPositions())); redactionLogEntry.setValue(manualResizeRedact.getValue()); redactionLogEntry.setHasBeenResized(true); @@ -225,8 +258,7 @@ public class RedactionLogMergeService { } else if (manualResizeRedact.getStatus().equals(AnnotationStatus.REQUESTED)) { manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to resize redact"); redactionLogEntry.setStatus(AnnotationStatus.REQUESTED); - redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry - .isRedacted(), false)); + redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted(), false)); redactionLogEntry.setPositions(convertPositions(manualResizeRedact.getPositions())); } else { redactionLogEntry.setStatus(AnnotationStatus.DECLINED); @@ -278,9 +310,13 @@ public class RedactionLogMergeService { return redactionLogEntries; } - private List convertPositions(List positions) { - return positions.stream().map(pos -> new Rectangle(new Point(pos.getTopLeftX(), pos.getTopLeftY()), pos.getWidth(), pos - .getHeight(), pos.getPage())).collect(Collectors.toList()); + + private List convertPositions( + List positions) { + + return positions.stream() + .map(pos -> new Rectangle(new Point(pos.getTopLeftX(), pos.getTopLeftY()), pos.getWidth(), pos.getHeight(), pos.getPage())) + .collect(Collectors.toList()); } @@ -311,6 +347,7 @@ public class RedactionLogMergeService { .status(manualRedactionEntry.getStatus()) .manualRedactionType(ManualRedactionType.ADD) .manualRedactionUserId(manualRedactionEntry.getUser()) + .rectangle(manualRedactionEntry.isRectangle()) .build(); } @@ -362,6 +399,5 @@ public class RedactionLogMergeService { } - } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java index d514df1e..7d101356 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java @@ -1,5 +1,17 @@ package com.iqser.red.service.redaction.v1.server.redaction.utils; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + import com.iqser.red.service.redaction.v1.model.Engine; import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary; import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrementValue; @@ -10,10 +22,6 @@ import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText; import lombok.experimental.UtilityClass; import lombok.extern.slf4j.Slf4j; -import java.util.*; -import java.util.regex.Pattern; -import java.util.stream.Collectors; - @Slf4j @UtilityClass @SuppressWarnings("PMD") @@ -37,8 +45,7 @@ public class EntitySearchUtils { startIndex = inputString.indexOf(cleanValue, stopIndex); stopIndex = startIndex + cleanValue.length(); - if (startIndex > -1 && (startIndex == 0 || Character.isWhitespace(inputString.charAt(startIndex - 1)) || isSeparator(inputString - .charAt(startIndex - 1))) && (stopIndex == inputString.length() || isSeparator(inputString.charAt(stopIndex)))) { + if (startIndex > -1 && (startIndex == 0 || Character.isWhitespace(inputString.charAt(startIndex - 1)) || isSeparator(inputString.charAt(startIndex - 1))) && (stopIndex == inputString.length() || isSeparator(inputString.charAt(stopIndex)))) { return true; } } while (startIndex > -1); @@ -66,8 +73,7 @@ public class EntitySearchUtils { startIndex = inputString.indexOf(cleanValue, stopIndex); stopIndex = startIndex + cleanValue.length(); - if (startIndex > -1 && (startIndex == 0 || Character.isWhitespace(inputString.charAt(startIndex - 1)) || isSeparator(inputString - .charAt(startIndex - 1))) && (stopIndex == inputString.length() || isSeparator(inputString.charAt(stopIndex)))) { + if (startIndex > -1 && (startIndex == 0 || Character.isWhitespace(inputString.charAt(startIndex - 1)) || isSeparator(inputString.charAt(startIndex - 1))) && (stopIndex == inputString.length() || isSeparator(inputString.charAt(stopIndex)))) { found.add(new Entity(inputString.substring(startIndex, stopIndex), type, startIndex, stopIndex, headline, sectionNumber, isDictionaryEntry, isDossierDictionary, engine)); } } while (startIndex > -1); @@ -121,8 +127,7 @@ public class EntitySearchUtils { for (Entity word : entities) { for (Entity inner : entities) { if (inner.getWord().length() < word.getWord() - .length() && inner.getStart() >= word.getStart() && inner.getEnd() <= word.getEnd() && word != inner && word - .getSectionNumber() == inner.getSectionNumber()) { + .length() && inner.getStart() >= word.getStart() && inner.getEnd() <= word.getEnd() && word != inner && word.getSectionNumber() == inner.getSectionNumber()) { wordsToRemove.add(inner); } } @@ -141,14 +146,14 @@ public class EntitySearchUtils { if (entities.contains(found)) { Optional existingOptional = entities.stream().filter(entity -> entity.equals(found)).findFirst(); - if(!existingOptional.isPresent()){ + if (!existingOptional.isPresent()) { return; } var existing = existingOptional.get(); - if(existing.getType().equals(found.getType())){ + if (existing.getType().equals(found.getType())) { existing.getEngines().addAll(found.getEngines()); - } else if (dictionary.getDictionaryRank(existing.getType()) <= dictionary.getDictionaryRank(found.getType())){ + } else if (dictionary.getDictionaryRank(existing.getType()) <= dictionary.getDictionaryRank(found.getType())) { entities.remove(found); entities.add(found); } @@ -165,12 +170,14 @@ public class EntitySearchUtils { } - public void addOrAddEngine(Set existing, Set toBeAdded){ + public void addOrAddEngine(Set existing, Set toBeAdded) { - for(Entity toAdd: toBeAdded){ + for (Entity toAdd : toBeAdded) { if (existing.contains(toAdd)) { - Optional existingOptional = existing.stream().filter(entity -> entity.equals(toAdd)).findFirst(); - if(!existingOptional.isPresent()){ + Optional existingOptional = existing.stream() + .filter(entity -> entity.equals(toAdd)) + .findFirst(); + if (!existingOptional.isPresent()) { return; } var existingEntity = existingOptional.get(); @@ -181,4 +188,20 @@ public class EntitySearchUtils { } } + + public Set findNonOverlappingMatchEntities(Set existingEntities, Set foundEntities) { + + Set result = new HashSet<>(); + if (existingEntities != null && foundEntities != null) { + for (Entity existingEntity : existingEntities) { + for (Entity foundEntity : foundEntities) { + if (existingEntity.getEnd() < foundEntity.getStart() || foundEntity.getEnd() < existingEntity.getStart()) { + result.add(foundEntity); + } + } + } + } + return result; + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtilsTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtilsTest.java index ae2fb019..eec7d419 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtilsTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtilsTest.java @@ -27,4 +27,164 @@ public class EntitySearchUtilsTest { } + + /* + * Text: Batman X. Superman Y. + * Position: 0123456789 + * 0123456789 + * 0123456789 + */ + @Test + public void testNotOverlappingEntitiesExpandedEnd() { + + // Arrange + Set existingEntities = new HashSet<>(); + Entity existingEntity1 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, false, false, Engine.RULE); + Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE); + existingEntities.add(existingEntity1); + existingEntities.add(existingEntity2); + + Set foundEntities = new HashSet<>(); + Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE); + Entity foundEntities2 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, false, false, Engine.RULE); + foundEntities.add(foundEntities1); + foundEntities.add(foundEntities2); + + // Act + Set result = EntitySearchUtils.findNonOverlappingMatchEntities(existingEntities, foundEntities); + + // Assert + assertThat(result.size()).isEqualTo(2); + assertThat(result).contains(foundEntities1); + assertThat(result).contains(foundEntities2); + + } + + + /* + * Text: Batman X. Superman Y. + * Position: 0123456789 + * 0123456789 + * 0123456789 + */ + @Test + public void testNotOverlappingEntitiesExpandedStartAndEndOverlapping() { + + // Arrange + Set existingEntities = new HashSet<>(); + Entity existingEntity1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE); + Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE); + existingEntities.add(existingEntity1); + existingEntities.add(existingEntity2); + + Set foundEntities = new HashSet<>(); + Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE); + Entity foundEntities2 = new Entity("X. Superman Y.", "fake type", 7, 20, "fake headline", 0, false, false, Engine.RULE); + foundEntities.add(foundEntities1); + foundEntities.add(foundEntities2); + + // Act + Set result = EntitySearchUtils.findNonOverlappingMatchEntities(existingEntities, foundEntities); + + // Assert + assertThat(result.size()).isEqualTo(1); + assertThat(result).contains(foundEntities1); + + } + + + /* + * Text: Batman X. Superman Y. + * Position: 0123456789 + * 0123456789 + * 0123456789 + */ + @Test + public void testNotOverlappingEntitiesExpandedStartAndEnd() { + + // Arrange + Set existingEntities = new HashSet<>(); + Entity existingEntity1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE); + Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE); + existingEntities.add(existingEntity1); + existingEntities.add(existingEntity2); + + Set foundEntities = new HashSet<>(); + Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE); + Entity foundEntities2 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, false, false, Engine.RULE); + foundEntities.add(foundEntities1); + foundEntities.add(foundEntities2); + + // Act + Set result = EntitySearchUtils.findNonOverlappingMatchEntities(existingEntities, foundEntities); + + // Assert + assertThat(result.size()).isEqualTo(1); + assertThat(result).contains(foundEntities1); + + } + + + /* + * Text: Batman X. Superman Y. + * Position: 0123456789 + * 0123456789 + * 0123456789 + */ + @Test + public void testNotOverlappingEntitiesExpandedExistingAndExpandedEnd() { + + // Arrange + Set existingEntities = new HashSet<>(); + Entity existingEntity1 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, false, false, Engine.RULE); + Entity existingEntity2 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, false, false, Engine.RULE); + existingEntities.add(existingEntity1); + existingEntities.add(existingEntity2); + + Set foundEntities = new HashSet<>(); + Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE); + Entity foundEntities2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE); + foundEntities.add(foundEntities1); + foundEntities.add(foundEntities2); + + // Act + Set result = EntitySearchUtils.findNonOverlappingMatchEntities(existingEntities, foundEntities); + + // Assert + assertThat(result.size()).isEqualTo(1); + assertThat(result).contains(foundEntities2); + + } + + /* + * Text: Batman X. Superman Y. + * Position: 0123456789 + * 0123456789 + * 0123456789 + */ + @Test + public void testNotOverlappingEntitiesExpandedEndLong() { + + // Arrange + Set existingEntities = new HashSet<>(); + Entity existingEntity1 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, false, false, Engine.RULE); + Entity existingEntity2 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, false, false, Engine.RULE); + existingEntities.add(existingEntity1); + existingEntities.add(existingEntity2); + + Set foundEntities = new HashSet<>(); + Entity foundEntities1 = new Entity("Batman X. Superman", "fake type", 0, 17, "fake headline", 0, false, false, Engine.RULE); + Entity foundEntities2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE); + foundEntities.add(foundEntities1); + foundEntities.add(foundEntities2); + + // Act + Set result = EntitySearchUtils.findNonOverlappingMatchEntities(existingEntities, foundEntities); + + // Assert + assertThat(result.size()).isEqualTo(1); + assertThat(result).contains(foundEntities2); + + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/CBI_author.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/CBI_author.txt index e2854b4c..9cf34080 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/CBI_author.txt +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/CBI_author.txt @@ -1,3 +1,6 @@ +Foo +F. Bar +B. Foo Johnson R | Weissler M S and Butters C A AD Hurt diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl index c0b96ff9..007fea23 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl @@ -19,8 +19,8 @@ rule "0: Expand CBI Authors with firstname initials" when Section(matchesType("CBI_author") || matchesType("recommendation_CBI_author")) then - section.expandByRegEx("CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1, "[^\\s]+"); - section.expandByRegEx("recommendation_CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1, "[^\\s]+"); + section.expandByRegEx("CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1, "[\\s]+"); + section.expandByRegEx("recommendation_CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1, "[\\s]+"); end @@ -344,4 +344,11 @@ rule "29: Redact Dossier Redactions" Section(matchesType("dossier_redactions")) then section.redact("dossier_redactions", 29, "Dossier Redaction found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); - end \ No newline at end of file + end + +rule "30: Ignore dossier_redactions if confidential" + when + Section(!fileAttributeByLabelEqualsIgnoreCase("Confidentiality","confidential") && matchesType("dossier_redactions")); + then + section.ignore("dossier_redactions"); + end \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/Minimal Examples/foo.pdf b/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/Minimal Examples/foo.pdf new file mode 100644 index 00000000..b6ddcbc0 Binary files /dev/null and b/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/Minimal Examples/foo.pdf differ