Pull request #413: RED-4301: Revert redaction log merge of duplicate annotations
Merge in RED/redaction-service from RED-4301 to master * commit '57ef7ead72018a91d9c15530efce40f713fb6c86': RED-4301: Revert redaction log merge of duplicate annotations
This commit is contained in:
Dominique Eiflaender
commit
c81b062784
@ -1,15 +1,40 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.service;
|
||||
|
||||
import java.awt.Color;
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.*;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualForceRedaction;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualLegalBasisChange;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualRedactionEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualResizeRedaction;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
|
||||
import com.iqser.red.service.redaction.v1.model.ManualChange;
|
||||
import com.iqser.red.service.redaction.v1.model.ManualRedactionType;
|
||||
import com.iqser.red.service.redaction.v1.model.Point;
|
||||
import com.iqser.red.service.redaction.v1.model.Rectangle;
|
||||
import com.iqser.red.service.redaction.v1.model.*;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLog;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLogComment;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionRequest;
|
||||
import com.iqser.red.service.redaction.v1.model.SectionArea;
|
||||
import com.iqser.red.service.redaction.v1.model.SectionGrid;
|
||||
import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
|
||||
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
|
||||
|
||||
@ -18,20 +43,12 @@ import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.awt.*;
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.List;
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class RedactionLogMergeService {
|
||||
|
||||
|
||||
private final SectionTextService sectionTextService;
|
||||
|
||||
private final RedactionStorageService redactionStorageService;
|
||||
@ -39,6 +56,7 @@ public class RedactionLogMergeService {
|
||||
|
||||
@Timed("redactmanager_getMergedRedactionLog")
|
||||
public RedactionLog provideRedactionLog(RedactionRequest redactionRequest) {
|
||||
|
||||
log.debug("Requested preview for: {}", redactionRequest);
|
||||
|
||||
var redactionLog = redactionStorageService.getRedactionLog(redactionRequest.getDossierId(), redactionRequest.getFileId());
|
||||
@ -72,22 +90,15 @@ public class RedactionLogMergeService {
|
||||
return merged;
|
||||
}
|
||||
|
||||
private RedactionLog mergeRedactionLogData(RedactionLog redactionLog, SectionGrid sectionGrid, ManualRedactions manualRedactions,
|
||||
Set<Integer> excludedPages, List<Type> types, Colors colors) {
|
||||
|
||||
private RedactionLog mergeRedactionLogData(RedactionLog redactionLog, SectionGrid sectionGrid,
|
||||
ManualRedactions manualRedactions, Set<Integer> excludedPages,
|
||||
List<Type> types, Colors colors) {
|
||||
|
||||
var skippedImportedRedactions = new HashSet<>();
|
||||
var validSourceIds = new HashSet<>();
|
||||
log.info("Merging Redaction log with manual redactions");
|
||||
if (manualRedactions != null) {
|
||||
|
||||
// entries that have been added to dictionaries
|
||||
validSourceIds.addAll(manualRedactions.getEntriesToAdd().stream()
|
||||
.filter(entry -> entry.getSourceId() != null)
|
||||
.filter(entry -> entry.isAddToDictionary() || entry.isAddToDossierDictionary())
|
||||
.filter(entry -> entry.getStatus() == AnnotationStatus.APPROVED)
|
||||
.filter(entry -> entry.getProcessedDate() != null)
|
||||
.map(ManualRedactionEntry::getSourceId).collect(Collectors.toList()));
|
||||
|
||||
var manualRedactionLogEntries = addManualAddEntries(sectionGrid, manualRedactions.getEntriesToAdd(), manualRedactions.getComments(), colors, types);
|
||||
|
||||
redactionLog.getRedactionLogEntry().addAll(manualRedactionLogEntries);
|
||||
@ -120,13 +131,11 @@ public class RedactionLogMergeService {
|
||||
Set<String> processedIds = new HashSet<>();
|
||||
redactionLog.getRedactionLogEntry().removeIf(entry -> {
|
||||
|
||||
if(validSourceIds.contains(entry.getId())){
|
||||
return true;
|
||||
}
|
||||
|
||||
if (entry.getImportedRedactionIntersections() != null) {
|
||||
entry.getImportedRedactionIntersections().removeAll(skippedImportedRedactions);
|
||||
if (!entry.getImportedRedactionIntersections().isEmpty() && (!entry.isImage() || entry.isImage() && !(entry.getType().equals("image") || entry.getType().equals("ocr")))) {
|
||||
if (!entry.getImportedRedactionIntersections()
|
||||
.isEmpty() && (!entry.isImage() || entry.isImage() && !(entry.getType()
|
||||
.equals("image") || entry.getType().equals("ocr")))) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@ -191,7 +200,6 @@ public class RedactionLogMergeService {
|
||||
String manualOverrideReason = null;
|
||||
if (imageRecategorization.getStatus().equals(AnnotationStatus.APPROVED)) {
|
||||
|
||||
|
||||
redactionLogEntry.setType(imageRecategorization.getType());
|
||||
redactionLogEntry.setSection("Image:" + redactionLogEntry.getType());
|
||||
|
||||
@ -210,14 +218,16 @@ public class RedactionLogMergeService {
|
||||
|
||||
redactionLogEntry.setReason(manualOverrideReason);
|
||||
|
||||
redactionLogEntry.getManualChanges().add(ManualChange.from(imageRecategorization)
|
||||
redactionLogEntry.getManualChanges()
|
||||
.add(ManualChange.from(imageRecategorization)
|
||||
.withManualRedactionType(ManualRedactionType.RECATEGORIZE)
|
||||
.withChange("type", imageRecategorization.getType()));
|
||||
}
|
||||
|
||||
if (mrw.getItem() instanceof IdRemoval) {
|
||||
var manualRemoval = (IdRemoval) mrw.getItem();
|
||||
if (manualRemoval.getStatus().equals(AnnotationStatus.APPROVED) && manualRemoval.isRemoveFromDictionary()) {
|
||||
if (manualRemoval.getStatus()
|
||||
.equals(AnnotationStatus.APPROVED) && manualRemoval.isRemoveFromDictionary()) {
|
||||
log.debug("Skipping merge for dictionary-modifying entry");
|
||||
} else {
|
||||
String manualOverrideReason = null;
|
||||
@ -233,9 +243,9 @@ public class RedactionLogMergeService {
|
||||
|
||||
redactionLogEntry.setReason(manualOverrideReason);
|
||||
|
||||
redactionLogEntry.getManualChanges().add(ManualChange.from(manualRemoval)
|
||||
.withManualRedactionType(manualRemoval.isRemoveFromDictionary() ?
|
||||
ManualRedactionType.REMOVE_FROM_DICTIONARY : ManualRedactionType.REMOVE_LOCALLY));
|
||||
redactionLogEntry.getManualChanges()
|
||||
.add(ManualChange.from(manualRemoval)
|
||||
.withManualRedactionType(manualRemoval.isRemoveFromDictionary() ? ManualRedactionType.REMOVE_FROM_DICTIONARY : ManualRedactionType.REMOVE_LOCALLY));
|
||||
}
|
||||
}
|
||||
|
||||
@ -308,7 +318,7 @@ public class RedactionLogMergeService {
|
||||
redactionLogEntry.setValue(manualResizeRedact.getValue());
|
||||
|
||||
// This is for backwards compatibility, now the text after/before is calculated during reanalysis because we need to find dict entries on positions where entries are resized to smaller.
|
||||
if(manualResizeRedact.getTextBefore() != null || manualResizeRedact.getTextAfter() != null) {
|
||||
if (manualResizeRedact.getTextBefore() != null || manualResizeRedact.getTextAfter() != null) {
|
||||
redactionLogEntry.setTextBefore(manualResizeRedact.getTextBefore());
|
||||
redactionLogEntry.setTextAfter(manualResizeRedact.getTextAfter());
|
||||
}
|
||||
@ -319,14 +329,15 @@ public class RedactionLogMergeService {
|
||||
redactionLogEntry.setPositions(convertPositions(manualResizeRedact.getPositions()));
|
||||
|
||||
// This is for backwards compatibility, now the text after/before is calculated during reanalysis because we need to find dict entries on positions where entries are resized to smaller.
|
||||
if(manualResizeRedact.getTextBefore() != null || manualResizeRedact.getTextAfter() != null) {
|
||||
if (manualResizeRedact.getTextBefore() != null || manualResizeRedact.getTextAfter() != null) {
|
||||
redactionLogEntry.setTextBefore(manualResizeRedact.getTextBefore());
|
||||
redactionLogEntry.setTextAfter(manualResizeRedact.getTextAfter());
|
||||
}
|
||||
}
|
||||
|
||||
redactionLogEntry.setReason(manualOverrideReason);
|
||||
redactionLogEntry.getManualChanges().add(ManualChange.from(manualResizeRedact)
|
||||
redactionLogEntry.getManualChanges()
|
||||
.add(ManualChange.from(manualResizeRedact)
|
||||
.withManualRedactionType(ManualRedactionType.RESIZE)
|
||||
.withChange("value", manualResizeRedact.getValue()));
|
||||
}
|
||||
@ -350,7 +361,8 @@ public class RedactionLogMergeService {
|
||||
|
||||
|
||||
public List<RedactionLogEntry> addManualAddEntries(SectionGrid sectionGrid, Set<ManualRedactionEntry> manualAdds,
|
||||
Map<String, List<Comment>> comments, Colors colors, List<Type> types) {
|
||||
Map<String, List<Comment>> comments, Colors colors,
|
||||
List<Type> types) {
|
||||
|
||||
List<RedactionLogEntry> redactionLogEntries = new ArrayList<>();
|
||||
|
||||
@ -372,9 +384,12 @@ public class RedactionLogMergeService {
|
||||
return redactionLogEntries;
|
||||
}
|
||||
|
||||
|
||||
private List<RedactionLogComment> convert(List<Comment> comments) {
|
||||
return comments == null ? null :comments.stream().map(c -> new RedactionLogComment(c.getId(), c.getUser(), c.getText(), c.getAnnotationId(),
|
||||
c.getFileId(), c.getDate(), c.getSoftDeletedTime())).collect(Collectors.toList());
|
||||
|
||||
return comments == null ? null : comments.stream()
|
||||
.map(c -> new RedactionLogComment(c.getId(), c.getUser(), c.getText(), c.getAnnotationId(), c.getFileId(), c.getDate(), c.getSoftDeletedTime()))
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
|
||||
@ -389,8 +404,8 @@ public class RedactionLogMergeService {
|
||||
|
||||
@SuppressWarnings("PMD.UselessParentheses")
|
||||
private boolean shouldCreateManualEntry(ManualRedactionEntry manualRedactionEntry) {
|
||||
return (!manualRedactionEntry.isAddToDictionary() && !manualRedactionEntry.isAddToDossierDictionary())
|
||||
|| ((manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDictionary()) && manualRedactionEntry.getProcessedDate() == null);
|
||||
|
||||
return (!manualRedactionEntry.isAddToDictionary() && !manualRedactionEntry.isAddToDossierDictionary()) || ((manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDictionary()) && manualRedactionEntry.getProcessedDate() == null);
|
||||
}
|
||||
|
||||
|
||||
@ -424,8 +439,8 @@ public class RedactionLogMergeService {
|
||||
}
|
||||
|
||||
|
||||
private float[] getColor(String type, Colors colors, boolean requested, boolean isRedaction,
|
||||
boolean skipped, List<Type> types) {
|
||||
private float[] getColor(String type, Colors colors, boolean requested, boolean isRedaction, boolean skipped,
|
||||
List<Type> types) {
|
||||
|
||||
if (requested) {
|
||||
return convertColor(colors.getRequestRemove());
|
||||
@ -456,7 +471,8 @@ public class RedactionLogMergeService {
|
||||
return convertColor(foundAndNotDeletedType.get().getHexColor());
|
||||
}
|
||||
Optional<Type> firstDeletedType = matchingTypes.stream().findFirst();
|
||||
return firstDeletedType.map(value -> convertColor(value.getHexColor())).orElseGet(() -> convertColor("#9398a0"));
|
||||
return firstDeletedType.map(value -> convertColor(value.getHexColor()))
|
||||
.orElseGet(() -> convertColor("#9398a0"));
|
||||
}
|
||||
|
||||
|
||||
@ -478,11 +494,15 @@ public class RedactionLogMergeService {
|
||||
return new float[]{color.getRed() / 255f, color.getGreen() / 255f, color.getBlue() / 255f};
|
||||
}
|
||||
|
||||
|
||||
private List<Type> getMatchingTypes(List<Type> types, String type) {
|
||||
|
||||
return types.stream().filter(t -> t.getType().equals(type)).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
|
||||
private boolean isDeletedType(Type type) {
|
||||
|
||||
return type.getSoftDeletedTime() != null;
|
||||
}
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user