Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-7834: fixes for migration

Browse Source 
* remove color
* fix manualChangeOverwrite
This commit is contained in:
Kilian Schuettler 2023-12-07 11:06:05 +01:00
parent 8fa833f873
commit 43b05d58a9
3 changed files with 68 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/LegacyRedactionLogMergeService.java
View File

@ -48,7 +48,7 @@ public class LegacyRedactionLogMergeService {
log.info("Merging Redaction log with manual redactions");
if (manualRedactions != null) {
var manualRedactionLogEntries = addManualAddEntries(manualRedactions.getEntriesToAdd(), dossierTemplateId, redactionLog.getAnalysisNumber());
var manualRedactionLogEntries = addManualAddEntries(manualRedactions.getEntriesToAdd(), redactionLog.getAnalysisNumber());
redactionLog.getRedactionLogEntry().addAll(manualRedactionLogEntries);
@ -137,7 +137,7 @@ public class LegacyRedactionLogMergeService {
}
if (item instanceof IdRemoval manualRemoval) {
processIdRemoval(redactionLogEntry, dossierTemplateId, manualRemoval);
processIdRemoval(redactionLogEntry, manualRemoval);
}
if (item instanceof ManualForceRedaction manualForceRedact) {
@ -145,11 +145,11 @@ public class LegacyRedactionLogMergeService {
}
if (item instanceof ManualLegalBasisChange manualLegalBasisChange) {
processManualLegalBasisChange(redactionLogEntry, dossierTemplateId, manualLegalBasisChange);
processManualLegalBasisChange(redactionLogEntry, manualLegalBasisChange);
}
if (item instanceof ManualResizeRedaction manualResizeRedact) {
processManualResizeRedaction(redactionLogEntry, dossierTemplateId, manualResizeRedact);
processManualResizeRedaction(redactionLogEntry, manualResizeRedact);
}
});
@ -175,7 +175,6 @@ public class LegacyRedactionLogMergeService {
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", recategorized by manual override");
} else if (imageRecategorization.getStatus().equals(AnnotationStatus.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to recategorize");
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted()));
}
if (manualOverrideReason != null) {
@ -200,22 +199,19 @@ public class LegacyRedactionLogMergeService {
}
private void processIdRemoval(RedactionLogEntry redactionLogEntry,String dossierTemplateId, IdRemoval manualRemoval) {
private void processIdRemoval(RedactionLogEntry redactionLogEntry, IdRemoval manualRemoval) {
boolean isApprovedRedaction = manualRemoval.getStatus().equals(AnnotationStatus.APPROVED);
if (isApprovedRedaction && manualRemoval.isRemoveFromDictionary() && isBasedOnDictionaryOnly(redactionLogEntry)) {
log.debug("Skipping merge for dictionary-modifying entry");
} else {
String redactionLogEntryType = redactionLogEntry.getType();
String manualOverrideReason = null;
if (isApprovedRedaction) {
redactionLogEntry.setRedacted(false);
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", removed by manual override");
redactionLogEntry.setColor(getColor(redactionLogEntryType, dossierTemplateId, false, redactionLogEntry.isRedacted()));
redactionLogEntry.setHint(false);
} else if (manualRemoval.getStatus().equals(AnnotationStatus.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to remove");
redactionLogEntry.setColor(getColor(redactionLogEntryType, dossierTemplateId, true, redactionLogEntry.isRedacted()));
}
if (manualOverrideReason != null) {
@ -246,12 +242,10 @@ public class LegacyRedactionLogMergeService {
} else {
redactionLogEntry.setRedacted(true);
}
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted()));
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", forced by manual override");
redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
} else if (manualForceRedact.getStatus().equals(AnnotationStatus.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to force " + (dictionaryIsHint ? "hint" : "redact"));
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted()));
redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
}
@ -265,7 +259,7 @@ public class LegacyRedactionLogMergeService {
}
private void processManualLegalBasisChange(RedactionLogEntry redactionLogEntry, String dossierTemplateId, ManualLegalBasisChange manualLegalBasisChange) {
private void processManualLegalBasisChange(RedactionLogEntry redactionLogEntry, ManualLegalBasisChange manualLegalBasisChange) {
String manualOverrideReason = null;
if (manualLegalBasisChange.getStatus().equals(AnnotationStatus.APPROVED)) {
@ -280,7 +274,6 @@ public class LegacyRedactionLogMergeService {
}
} else if (manualLegalBasisChange.getStatus().equals(AnnotationStatus.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis change requested");
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted()));
}
if (manualOverrideReason != null) {
@ -299,11 +292,10 @@ public class LegacyRedactionLogMergeService {
}
private void processManualResizeRedaction(RedactionLogEntry redactionLogEntry, String dossierTemplateId, ManualResizeRedaction manualResizeRedact) {
private void processManualResizeRedaction(RedactionLogEntry redactionLogEntry, ManualResizeRedaction manualResizeRedact) {
String manualOverrideReason = null;
if (manualResizeRedact.getStatus().equals(AnnotationStatus.APPROVED)) {
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted()));
redactionLogEntry.setPositions(convertPositions(manualResizeRedact.getPositions()));
if (!"signature".equalsIgnoreCase(redactionLogEntry.getType()) && !"logo".equalsIgnoreCase(redactionLogEntry.getType())) {
redactionLogEntry.setValue(manualResizeRedact.getValue());
@ -316,7 +308,6 @@ public class LegacyRedactionLogMergeService {
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", resized by manual override");
} else if (manualResizeRedact.getStatus().equals(AnnotationStatus.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to resize redact");
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted()));
redactionLogEntry.setPositions(convertPositions(manualResizeRedact.getPositions()));
// This is for backwards compatibility, now the text after/before is calculated during reanalysis because we need to find dict entries on positions where entries are resized to smaller.
@ -332,14 +323,14 @@ public class LegacyRedactionLogMergeService {
}
public List<RedactionLogEntry> addManualAddEntries(Set<ManualRedactionEntry> manualAdds, String dossierTemplateId, int analysisNumber) {
public List<RedactionLogEntry> addManualAddEntries(Set<ManualRedactionEntry> manualAdds, int analysisNumber) {
List<RedactionLogEntry> redactionLogEntries = new ArrayList<>();
for (ManualRedactionEntry manualRedactionEntry : manualAdds) {
if (shouldCreateManualEntry(manualRedactionEntry)) {
RedactionLogEntry redactionLogEntry = createRedactionLogEntry(manualRedactionEntry, manualRedactionEntry.getAnnotationId(), dossierTemplateId, analysisNumber);
RedactionLogEntry redactionLogEntry = createRedactionLogEntry(manualRedactionEntry, manualRedactionEntry.getAnnotationId(), analysisNumber);
redactionLogEntry.setPositions(convertPositions(manualRedactionEntry.getPositions()));
redactionLogEntry.setTextBefore(manualRedactionEntry.getTextBefore());
redactionLogEntry.setTextAfter(manualRedactionEntry.getTextAfter());
@ -367,7 +358,7 @@ public class LegacyRedactionLogMergeService {
}
private RedactionLogEntry createRedactionLogEntry(ManualRedactionEntry manualRedactionEntry, String id, String dossierTemplateId, int analysisNumber) {
private RedactionLogEntry createRedactionLogEntry(ManualRedactionEntry manualRedactionEntry, String id, int analysisNumber) {
var addToDictionary = manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary();
@ -377,7 +368,6 @@ public class LegacyRedactionLogMergeService {
return RedactionLogEntry.builder()
.id(id)
.color(getColorForManualAdd(manualRedactionEntry.getType(), dossierTemplateId, manualRedactionEntry.getStatus()))
.reason(manualRedactionEntry.getReason())
.isDictionaryEntry(manualRedactionEntry.isAddToDictionary())
.isDossierDictionaryEntry(manualRedactionEntry.isAddToDossierDictionary())
@ -396,30 +386,6 @@ public class LegacyRedactionLogMergeService {
}
private float[] getColor(String type, String dossierTemplateId, boolean isRedaction, boolean skipped) {
if (skipped || !isRedaction && !isHint(dossierTemplateId, type)) {
return dictionaryService.getNotRedactedColor(dossierTemplateId);
}
return dictionaryService.getColor(type, dossierTemplateId);
}
private float[] getColorForManualAdd(String type, String dossierTemplateId, AnnotationStatus status) {
if (status.equals(AnnotationStatus.REQUESTED) || status.equals(AnnotationStatus.DECLINED)) {
return dictionaryService.getNotRedactedColor(dossierTemplateId);
}
return dictionaryService.getColor(type, dossierTemplateId);
}
boolean isHint(String dossierTemplateId, String type) {
return dictionaryService.isHint(type, dossierTemplateId);
}
@Data
@AllArgsConstructor
private static class ManualRedactionWrapper implements Comparable<ManualRedactionWrapper> {

6
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigrationEntity.java
View File

@ -20,6 +20,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlo
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite;
import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
@ -51,7 +52,7 @@ public final class MigrationEntity {
String ruleIdentifier = "OLD." + redactionLogEntry.getMatchedRule() + ".0";
List<RectangleWithPage> rectangleWithPages = redactionLogEntry.getPositions().stream().map(RectangleWithPage::fromRedactionLogRectangle).toList();
EntityType entityType = getEntityType(redactionLogEntry);
return ManualEntity.builder()
.id(redactionLogEntry.getId())
.value(redactionLogEntry.getValue())
@ -61,11 +62,12 @@ public final class MigrationEntity {
.legalBasis(redactionLogEntry.getLegalBasis())
.type(redactionLogEntry.getType())
.section(redactionLogEntry.getSection())
.entityType(getEntityType(redactionLogEntry))
.entityType(entityType)
.applied(redactionLogEntry.isRedacted())
.isDictionaryEntry(redactionLogEntry.isDictionaryEntry())
.isDossierDictionaryEntry(redactionLogEntry.isDossierDictionaryEntry())
.rectangle(redactionLogEntry.isRectangle())
.manualOverwrite(new ManualChangeOverwrite(entityType))
.build();
}

58
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionLogCrawler.java
View File

@ -2,13 +2,18 @@ package com.iqser.red.service.redaction.v1.server;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.util.HashSet;
import java.util.Set;
import org.junit.jupiter.api.Test;
import com.iqser.red.commons.jackson.ObjectMapperFactory;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
import lombok.SneakyThrows;
@ -19,24 +24,69 @@ public class RedactionLogCrawler {
public void findRedactionLogVersion0() {
String redactionLogsDir = "/home/kschuettler/Nextcloud/redaction_logs/";
Files.walk(Path.of(redactionLogsDir))
long version0Count = Files.walk(Path.of(redactionLogsDir))
.parallel()
.filter(file -> {
try {
return Files.size(file) > 0;
} catch (IOException e) {
throw new RuntimeException(e);
}
})
.map(Path::toFile)
.filter(File::isFile)
.filter(file -> file.toString().endsWith("redactionlog.json"))
.peek(System.out::println)
.map(this::parseRedactionLog)
.filter(log -> log.getAnalysisVersion() == 0)
.filter(LogAndFile::isValid)
.filter(LogAndFile::hasAnyDuplicates)
.peek(LogAndFile::saveToTmp)
.count();
System.out.println(version0Count + " version 0 redaction logs found");
}
@SneakyThrows
private RedactionLog parseRedactionLog(File file) {
private LogAndFile parseRedactionLog(File file) {
var mapper = ObjectMapperFactory.create();
try (var in = new FileInputStream(file)) {
return mapper.readValue(in.readAllBytes(), RedactionLog.class);
return new LogAndFile(mapper.readValue(in.readAllBytes(), RedactionLog.class), file);
}
}
private record LogAndFile(RedactionLog redactionLog, File file) {
public boolean hasAnyDuplicates() {
Set<String> uniqueStrings = new HashSet<>();
for (String str : redactionLog.getRedactionLogEntry().stream().map(RedactionLogEntry::getId).toList()) {
if (!uniqueStrings.add(str)) {
return true;
}
}
return false;
}
@SneakyThrows
public void saveToTmp() {
Path original = file.toPath();
Path tmpFile = Path.of("/tmp/").resolve(Path.of("/home/kschuettler/Nextcloud").relativize(original));
tmpFile.toFile().mkdirs();
Files.copy(original, tmpFile, StandardCopyOption.REPLACE_EXISTING);
}
public boolean isValid() {
return redactionLog.getRedactionLogEntry() != null;
}
}
}