From 57ef7ead72018a91d9c15530efce40f713fb6c86 Mon Sep 17 00:00:00 2001 From: deiflaender Date: Mon, 20 Jun 2022 12:38:57 +0200 Subject: [PATCH] RED-4301: Revert redaction log merge of duplicate annotations --- .../service/RedactionLogMergeService.java | 114 ++++++++++-------- 1 file changed, 67 insertions(+), 47 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java index 34d6c872..98eb02e4 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java @@ -1,15 +1,40 @@ package com.iqser.red.service.redaction.v1.server.redaction.service; +import java.awt.Color; +import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +import org.springframework.stereotype.Service; + import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus; import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment; import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions; -import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.*; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualForceRedaction; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualLegalBasisChange; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualRedactionEntry; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualResizeRedaction; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type; +import com.iqser.red.service.redaction.v1.model.ManualChange; +import com.iqser.red.service.redaction.v1.model.ManualRedactionType; import com.iqser.red.service.redaction.v1.model.Point; import com.iqser.red.service.redaction.v1.model.Rectangle; -import com.iqser.red.service.redaction.v1.model.*; +import com.iqser.red.service.redaction.v1.model.RedactionLog; +import com.iqser.red.service.redaction.v1.model.RedactionLogComment; +import com.iqser.red.service.redaction.v1.model.RedactionLogEntry; +import com.iqser.red.service.redaction.v1.model.RedactionRequest; +import com.iqser.red.service.redaction.v1.model.SectionArea; +import com.iqser.red.service.redaction.v1.model.SectionGrid; import com.iqser.red.service.redaction.v1.server.exception.NotFoundException; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; @@ -18,20 +43,12 @@ import lombok.AllArgsConstructor; import lombok.Data; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Service; - -import java.awt.*; -import java.time.OffsetDateTime; -import java.util.List; -import java.util.*; -import java.util.stream.Collectors; @Slf4j @Service @RequiredArgsConstructor public class RedactionLogMergeService { - private final SectionTextService sectionTextService; private final RedactionStorageService redactionStorageService; @@ -39,6 +56,7 @@ public class RedactionLogMergeService { @Timed("redactmanager_getMergedRedactionLog") public RedactionLog provideRedactionLog(RedactionRequest redactionRequest) { + log.debug("Requested preview for: {}", redactionRequest); var redactionLog = redactionStorageService.getRedactionLog(redactionRequest.getDossierId(), redactionRequest.getFileId()); @@ -72,22 +90,15 @@ public class RedactionLogMergeService { return merged; } - private RedactionLog mergeRedactionLogData(RedactionLog redactionLog, SectionGrid sectionGrid, ManualRedactions manualRedactions, - Set excludedPages, List types, Colors colors) { + + private RedactionLog mergeRedactionLogData(RedactionLog redactionLog, SectionGrid sectionGrid, + ManualRedactions manualRedactions, Set excludedPages, + List types, Colors colors) { var skippedImportedRedactions = new HashSet<>(); - var validSourceIds = new HashSet<>(); log.info("Merging Redaction log with manual redactions"); if (manualRedactions != null) { - // entries that have been added to dictionaries - validSourceIds.addAll(manualRedactions.getEntriesToAdd().stream() - .filter(entry -> entry.getSourceId() != null) - .filter(entry -> entry.isAddToDictionary() || entry.isAddToDossierDictionary()) - .filter(entry -> entry.getStatus() == AnnotationStatus.APPROVED) - .filter(entry -> entry.getProcessedDate() != null) - .map(ManualRedactionEntry::getSourceId).collect(Collectors.toList())); - var manualRedactionLogEntries = addManualAddEntries(sectionGrid, manualRedactions.getEntriesToAdd(), manualRedactions.getComments(), colors, types); redactionLog.getRedactionLogEntry().addAll(manualRedactionLogEntries); @@ -120,13 +131,11 @@ public class RedactionLogMergeService { Set processedIds = new HashSet<>(); redactionLog.getRedactionLogEntry().removeIf(entry -> { - if(validSourceIds.contains(entry.getId())){ - return true; - } - if (entry.getImportedRedactionIntersections() != null) { entry.getImportedRedactionIntersections().removeAll(skippedImportedRedactions); - if (!entry.getImportedRedactionIntersections().isEmpty() && (!entry.isImage() || entry.isImage() && !(entry.getType().equals("image") || entry.getType().equals("ocr")))) { + if (!entry.getImportedRedactionIntersections() + .isEmpty() && (!entry.isImage() || entry.isImage() && !(entry.getType() + .equals("image") || entry.getType().equals("ocr")))) { return true; } } @@ -191,7 +200,6 @@ public class RedactionLogMergeService { String manualOverrideReason = null; if (imageRecategorization.getStatus().equals(AnnotationStatus.APPROVED)) { - redactionLogEntry.setType(imageRecategorization.getType()); redactionLogEntry.setSection("Image:" + redactionLogEntry.getType()); @@ -210,14 +218,16 @@ public class RedactionLogMergeService { redactionLogEntry.setReason(manualOverrideReason); - redactionLogEntry.getManualChanges().add(ManualChange.from(imageRecategorization) - .withManualRedactionType(ManualRedactionType.RECATEGORIZE) - .withChange("type", imageRecategorization.getType())); + redactionLogEntry.getManualChanges() + .add(ManualChange.from(imageRecategorization) + .withManualRedactionType(ManualRedactionType.RECATEGORIZE) + .withChange("type", imageRecategorization.getType())); } if (mrw.getItem() instanceof IdRemoval) { var manualRemoval = (IdRemoval) mrw.getItem(); - if (manualRemoval.getStatus().equals(AnnotationStatus.APPROVED) && manualRemoval.isRemoveFromDictionary()) { + if (manualRemoval.getStatus() + .equals(AnnotationStatus.APPROVED) && manualRemoval.isRemoveFromDictionary()) { log.debug("Skipping merge for dictionary-modifying entry"); } else { String manualOverrideReason = null; @@ -233,9 +243,9 @@ public class RedactionLogMergeService { redactionLogEntry.setReason(manualOverrideReason); - redactionLogEntry.getManualChanges().add(ManualChange.from(manualRemoval) - .withManualRedactionType(manualRemoval.isRemoveFromDictionary() ? - ManualRedactionType.REMOVE_FROM_DICTIONARY : ManualRedactionType.REMOVE_LOCALLY)); + redactionLogEntry.getManualChanges() + .add(ManualChange.from(manualRemoval) + .withManualRedactionType(manualRemoval.isRemoveFromDictionary() ? ManualRedactionType.REMOVE_FROM_DICTIONARY : ManualRedactionType.REMOVE_LOCALLY)); } } @@ -308,7 +318,7 @@ public class RedactionLogMergeService { redactionLogEntry.setValue(manualResizeRedact.getValue()); // This is for backwards compatibility, now the text after/before is calculated during reanalysis because we need to find dict entries on positions where entries are resized to smaller. - if(manualResizeRedact.getTextBefore() != null || manualResizeRedact.getTextAfter() != null) { + if (manualResizeRedact.getTextBefore() != null || manualResizeRedact.getTextAfter() != null) { redactionLogEntry.setTextBefore(manualResizeRedact.getTextBefore()); redactionLogEntry.setTextAfter(manualResizeRedact.getTextAfter()); } @@ -319,16 +329,17 @@ public class RedactionLogMergeService { redactionLogEntry.setPositions(convertPositions(manualResizeRedact.getPositions())); // This is for backwards compatibility, now the text after/before is calculated during reanalysis because we need to find dict entries on positions where entries are resized to smaller. - if(manualResizeRedact.getTextBefore() != null || manualResizeRedact.getTextAfter() != null) { + if (manualResizeRedact.getTextBefore() != null || manualResizeRedact.getTextAfter() != null) { redactionLogEntry.setTextBefore(manualResizeRedact.getTextBefore()); redactionLogEntry.setTextAfter(manualResizeRedact.getTextAfter()); } } redactionLogEntry.setReason(manualOverrideReason); - redactionLogEntry.getManualChanges().add(ManualChange.from(manualResizeRedact) - .withManualRedactionType(ManualRedactionType.RESIZE) - .withChange("value", manualResizeRedact.getValue())); + redactionLogEntry.getManualChanges() + .add(ManualChange.from(manualResizeRedact) + .withManualRedactionType(ManualRedactionType.RESIZE) + .withChange("value", manualResizeRedact.getValue())); } }); @@ -350,7 +361,8 @@ public class RedactionLogMergeService { public List addManualAddEntries(SectionGrid sectionGrid, Set manualAdds, - Map> comments, Colors colors, List types) { + Map> comments, Colors colors, + List types) { List redactionLogEntries = new ArrayList<>(); @@ -372,9 +384,12 @@ public class RedactionLogMergeService { return redactionLogEntries; } + private List convert(List comments) { - return comments == null ? null :comments.stream().map(c -> new RedactionLogComment(c.getId(), c.getUser(), c.getText(), c.getAnnotationId(), - c.getFileId(), c.getDate(), c.getSoftDeletedTime())).collect(Collectors.toList()); + + return comments == null ? null : comments.stream() + .map(c -> new RedactionLogComment(c.getId(), c.getUser(), c.getText(), c.getAnnotationId(), c.getFileId(), c.getDate(), c.getSoftDeletedTime())) + .collect(Collectors.toList()); } @@ -389,8 +404,8 @@ public class RedactionLogMergeService { @SuppressWarnings("PMD.UselessParentheses") private boolean shouldCreateManualEntry(ManualRedactionEntry manualRedactionEntry) { - return (!manualRedactionEntry.isAddToDictionary() && !manualRedactionEntry.isAddToDossierDictionary()) - || ((manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDictionary()) && manualRedactionEntry.getProcessedDate() == null); + + return (!manualRedactionEntry.isAddToDictionary() && !manualRedactionEntry.isAddToDossierDictionary()) || ((manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDictionary()) && manualRedactionEntry.getProcessedDate() == null); } @@ -424,8 +439,8 @@ public class RedactionLogMergeService { } - private float[] getColor(String type, Colors colors, boolean requested, boolean isRedaction, - boolean skipped, List types) { + private float[] getColor(String type, Colors colors, boolean requested, boolean isRedaction, boolean skipped, + List types) { if (requested) { return convertColor(colors.getRequestRemove()); @@ -456,7 +471,8 @@ public class RedactionLogMergeService { return convertColor(foundAndNotDeletedType.get().getHexColor()); } Optional firstDeletedType = matchingTypes.stream().findFirst(); - return firstDeletedType.map(value -> convertColor(value.getHexColor())).orElseGet(() -> convertColor("#9398a0")); + return firstDeletedType.map(value -> convertColor(value.getHexColor())) + .orElseGet(() -> convertColor("#9398a0")); } @@ -478,11 +494,15 @@ public class RedactionLogMergeService { return new float[]{color.getRed() / 255f, color.getGreen() / 255f, color.getBlue() / 255f}; } + private List getMatchingTypes(List types, String type) { + return types.stream().filter(t -> t.getType().equals(type)).collect(Collectors.toList()); } + private boolean isDeletedType(Type type) { + return type.getSoftDeletedTime() != null; }