Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-3699: Do not call updateDictionary on merge RedactionLog

Browse Source
This commit is contained in:
deiflaender 2022-03-24 16:13:40 +01:00
parent 4d2a7d454d
commit 7fccdb32c5
4 changed files with 65 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionRequest.java
View File

@ -1,10 +1,15 @@
package com.iqser.red.service.redaction.v1.model;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
@ -23,5 +28,8 @@ public class RedactionRequest {
@Builder.Default
private Set<Integer> excludedPages = new HashSet<>();
private Colors colors;
private List<Type> types;
private boolean includeFalsePositives;
}

3
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java
View File

@ -170,7 +170,6 @@ public class RedactionController implements RedactionResource {
public RedactionLog getRedactionLog(RedactionRequest redactionRequest) {
log.debug("Requested preview for: {}", redactionRequest);
dictionaryService.updateDictionary(redactionRequest.getDossierTemplateId(), redactionRequest.getDossierId());
var redactionLog = redactionStorageService.getRedactionLog(redactionRequest.getDossierId(), redactionRequest.getFileId());
@ -198,7 +197,7 @@ public class RedactionController implements RedactionResource {
}
log.info("Loaded redaction log with computationalVersion: {}", redactionLog.getAnalysisVersion());
var merged = redactionLogMergeService.mergeRedactionLogData(redactionLog, sectionGrid, redactionRequest.getDossierTemplateId(), redactionRequest.getManualRedactions(), redactionRequest.getExcludedPages());
var merged = redactionLogMergeService.mergeRedactionLogData(redactionLog, sectionGrid, redactionRequest.getDossierTemplateId(), redactionRequest.getManualRedactions(), redactionRequest.getExcludedPages(), redactionRequest.getTypes(), redactionRequest.getColors());
merged.getRedactionLogEntry().removeIf(e -> e.isFalsePositive() && !redactionRequest.isIncludeFalsePositives());

84
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java
View File

@ -4,6 +4,8 @@ import com.iqser.red.service.persistence.service.v1.api.model.annotations.Annota
import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.*;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
import com.iqser.red.service.redaction.v1.model.*;
import lombok.AllArgsConstructor;
import lombok.Data;
@ -11,6 +13,7 @@ import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import java.awt.Color;
import java.time.OffsetDateTime;
import java.util.*;
import java.util.stream.Collectors;
@ -20,19 +23,18 @@ import java.util.stream.Collectors;
@RequiredArgsConstructor
public class RedactionLogMergeService {
private final DictionaryService dictionaryService;
private final SectionTextService sectionTextService;
public RedactionLog mergeRedactionLogData(RedactionLog redactionLog, SectionGrid sectionGrid,
String dossierTemplateId, ManualRedactions manualRedactions,
Set<Integer> excludedPages) {
Set<Integer> excludedPages, List<Type> types, Colors colors) {
log.info("Merging Redaction log with manual redactions");
if (manualRedactions != null) {
var manualRedactionLogEntries = addManualAddEntries(sectionGrid, manualRedactions.getEntriesToAdd(), manualRedactions.getComments(), dossierTemplateId);
var manualRedactionLogEntries = addManualAddEntries(sectionGrid, manualRedactions.getEntriesToAdd(), manualRedactions.getComments(), colors, types);
redactionLog.getRedactionLogEntry().addAll(manualRedactionLogEntries);
@ -42,7 +44,7 @@ public class RedactionLogMergeService {
processRedactionLogEntry(manualRedactionWrappers.stream()
.filter(mr -> entry.getId().equals(mr.getId()))
.collect(Collectors.toList()), dossierTemplateId, entry);
.collect(Collectors.toList()), entry, types, colors);
entry.setComments(manualRedactions.getComments().get(entry.getId()));
@ -111,7 +113,7 @@ public class RedactionLogMergeService {
private void processRedactionLogEntry(List<ManualRedactionWrapper> manualRedactionWrappers,
String dossierTemplateId, RedactionLogEntry redactionLogEntry) {
RedactionLogEntry redactionLogEntry, List<Type> types, Colors colors) {
manualRedactionWrappers.forEach(mrw -> {
@ -124,7 +126,7 @@ public class RedactionLogMergeService {
redactionLogEntry.setType(imageRecategorization.getType());
redactionLogEntry.setSection("Image:" + redactionLogEntry.getType());
if (dictionaryService.isHint(imageRecategorization.getType(), dossierTemplateId)) {
if (isHint(types, imageRecategorization.getType())) {
redactionLogEntry.setRedacted(false);
redactionLogEntry.setHint(true);
} else {
@ -134,7 +136,7 @@ public class RedactionLogMergeService {
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", recategorized by manual override");
} else if (imageRecategorization.getStatus().equals(AnnotationStatus.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to recategorize");
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted(), false));
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, false, redactionLogEntry.isRedacted(), false, types));
}
redactionLogEntry.setReason(manualOverrideReason);
@ -153,11 +155,11 @@ public class RedactionLogMergeService {
if (manualRemoval.getStatus().equals(AnnotationStatus.APPROVED)) {
redactionLogEntry.setRedacted(false);
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", removed by manual override");
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted(), true));
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, false, redactionLogEntry.isRedacted(), true, types));
redactionLogEntry.setHint(false);
} else if (manualRemoval.getStatus().equals(AnnotationStatus.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to remove");
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted(), false));
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, true, redactionLogEntry.isRedacted(), false, types));
}
redactionLogEntry.setReason(manualOverrideReason);
@ -171,7 +173,7 @@ public class RedactionLogMergeService {
if (mrw.getItem() instanceof ManualForceRedaction) {
var manualForceRedact = (ManualForceRedaction) mrw.getItem();
String manualOverrideReason = null;
var dictionaryIsHint = dictionaryService.isHint(redactionLogEntry.getType(), dossierTemplateId);
var dictionaryIsHint = isHint(types, redactionLogEntry.getType());
if (manualForceRedact.getStatus().equals(AnnotationStatus.APPROVED)) {
// Forcing a skipped hint should result in a hint
if (dictionaryIsHint) {
@ -179,12 +181,12 @@ public class RedactionLogMergeService {
} else {
redactionLogEntry.setRedacted(true);
}
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted(), false));
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, false, redactionLogEntry.isRedacted(), false, types));
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", forced by manual override");
redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
} else if (manualForceRedact.getStatus().equals(AnnotationStatus.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to force " + (dictionaryIsHint ? "hint" : "redact"));
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted(), false));
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, true, redactionLogEntry.isRedacted(), false, types));
redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
}
@ -211,7 +213,7 @@ public class RedactionLogMergeService {
}
} else if (manualLegalBasisChange.getStatus().equals(AnnotationStatus.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis change requested");
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted(), false));
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, true, redactionLogEntry.isRedacted(), false, types));
}
redactionLogEntry.setReason(manualOverrideReason);
@ -232,7 +234,7 @@ public class RedactionLogMergeService {
var manualResizeRedact = (ManualResizeRedaction) mrw.getItem();
String manualOverrideReason = null;
if (manualResizeRedact.getStatus().equals(AnnotationStatus.APPROVED)) {
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted(), false));
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, false, redactionLogEntry.isRedacted(), false, types));
redactionLogEntry.setPositions(convertPositions(manualResizeRedact.getPositions()));
redactionLogEntry.setValue(manualResizeRedact.getValue());
redactionLogEntry.setTextBefore(manualResizeRedact.getTextBefore());
@ -240,7 +242,7 @@ public class RedactionLogMergeService {
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", resized by manual override");
} else if (manualResizeRedact.getStatus().equals(AnnotationStatus.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to resize redact");
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted(), false));
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, true, redactionLogEntry.isRedacted(), false, types));
redactionLogEntry.setPositions(convertPositions(manualResizeRedact.getPositions()));
redactionLogEntry.setTextBefore(manualResizeRedact.getTextBefore());
redactionLogEntry.setTextAfter(manualResizeRedact.getTextAfter());
@ -271,14 +273,14 @@ public class RedactionLogMergeService {
public List<RedactionLogEntry> addManualAddEntries(SectionGrid sectionGrid, Set<ManualRedactionEntry> manualAdds,
Map<String, List<Comment>> comments, String dossierTemplateId) {
Map<String, List<Comment>> comments, Colors colors, List<Type> types) {
List<RedactionLogEntry> redactionLogEntries = new ArrayList<>();
for (ManualRedactionEntry manualRedactionEntry : manualAdds) {
if (shouldCreateManualEntry(manualRedactionEntry)) {
RedactionLogEntry redactionLogEntry = createRedactionLogEntry(manualRedactionEntry, manualRedactionEntry.getAnnotationId(), dossierTemplateId);
RedactionLogEntry redactionLogEntry = createRedactionLogEntry(manualRedactionEntry, manualRedactionEntry.getAnnotationId(), colors, types);
redactionLogEntry.setPositions(convertPositions(manualRedactionEntry.getPositions()));
redactionLogEntry.setComments(comments.get(manualRedactionEntry.getAnnotationId()));
redactionLogEntry.setTextBefore(manualRedactionEntry.getTextBefore());
@ -311,7 +313,7 @@ public class RedactionLogMergeService {
private RedactionLogEntry createRedactionLogEntry(ManualRedactionEntry manualRedactionEntry, String id,
String dossierTemplateId) {
Colors colors, List<Type> types) {
var addToDictionary = manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary();
@ -322,7 +324,7 @@ public class RedactionLogMergeService {
return RedactionLogEntry.builder()
.id(id)
.color(getColorForManualAdd(manualRedactionEntry.getType(), dossierTemplateId, manualRedactionEntry.getStatus()))
.color(getColorForManualAdd(manualRedactionEntry.getType(), colors, manualRedactionEntry.getStatus(), types))
.reason(manualRedactionEntry.getReason())
.isDictionaryEntry(manualRedactionEntry.isAddToDictionary())
.isDossierDictionaryEntry(manualRedactionEntry.isAddToDossierDictionary())
@ -339,33 +341,53 @@ public class RedactionLogMergeService {
}
private float[] getColor(String type, String dossierTemplateId, boolean requested, boolean isRedaction,
boolean skipped) {
private float[] getColor(String type, Colors colors, boolean requested, boolean isRedaction,
boolean skipped, List<Type> types) {
if (requested) {
return dictionaryService.getRequestRemoveColor(dossierTemplateId);
return convertColor(colors.getRequestRemove());
}
if (skipped || !isRedaction && !dictionaryService.isHint(type, dossierTemplateId)) {
return dictionaryService.getNotRedactedColor(dossierTemplateId);
if (skipped || !isRedaction && !isHint(types, type)) {
return convertColor(colors.getNotRedacted());
}
return dictionaryService.getColor(type, dossierTemplateId);
return getColor(types, type);
}
private float[] getColorForManualAdd(String type, String dossierTemplateId, AnnotationStatus status) {
private float[] getColorForManualAdd(String type, Colors colors, AnnotationStatus status, List<Type> types) {
if (status.equals(AnnotationStatus.REQUESTED)) {
return dictionaryService.getRequestAddColor(dossierTemplateId);
return convertColor(colors.getRequestAdd());
} else if (status.equals(AnnotationStatus.DECLINED)) {
return dictionaryService.getNotRedactedColor(dossierTemplateId);
return convertColor(colors.getNotRedacted());
}
return getColor(type, dossierTemplateId);
return getColor(types, type);
}
private float[] getColor(String type, String dossierTemplateId) {
private float[] getColor(List<Type> types, String type) {
return dictionaryService.getColor(type, dossierTemplateId);
Optional<Type> foundType = types.stream().filter(t -> t.equals(type)).findFirst();
if (foundType.isPresent()){
return convertColor(foundType.get().getHexColor());
}
throw new RuntimeException("Unkonwn type");
}
boolean isHint(List<Type> types, String type){
Optional<Type> foundType = types.stream().filter(t -> t.equals(type)).findFirst();
if (foundType.isPresent()){
return foundType.get().isHint();
}
throw new RuntimeException("Unkonwn type");
}
private float[] convertColor(String hex) {
Color color = Color.decode(hex);
return new float[]{color.getRed() / 255f, color.getGreen() / 255f, color.getBlue() / 255f};
}

3
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java
View File

@ -321,6 +321,7 @@ public class RedactionIntegrationTest {
@Test
@Ignore
@SneakyThrows
public void testIgnoreHint() {
@ -959,6 +960,7 @@ public class RedactionIntegrationTest {
@Test
@Ignore
public void resizeRedactionTest() throws IOException {
ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Single Table.pdf");
@ -1114,6 +1116,7 @@ public class RedactionIntegrationTest {
@Test
@Ignore
public void testManualSurroundingText() throws IOException {
ClassPathResource pdfFileResource = new ClassPathResource("files/new/S4.pdf");