diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/LegacyRedactionLogMergeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/LegacyRedactionLogMergeService.java index 85969ff0..7098e491 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/LegacyRedactionLogMergeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/LegacyRedactionLogMergeService.java @@ -48,7 +48,7 @@ public class LegacyRedactionLogMergeService { log.info("Merging Redaction log with manual redactions"); if (manualRedactions != null) { - var manualRedactionLogEntries = addManualAddEntries(manualRedactions.getEntriesToAdd(), dossierTemplateId, redactionLog.getAnalysisNumber()); + var manualRedactionLogEntries = addManualAddEntries(manualRedactions.getEntriesToAdd(), redactionLog.getAnalysisNumber()); redactionLog.getRedactionLogEntry().addAll(manualRedactionLogEntries); @@ -137,7 +137,7 @@ public class LegacyRedactionLogMergeService { } if (item instanceof IdRemoval manualRemoval) { - processIdRemoval(redactionLogEntry, dossierTemplateId, manualRemoval); + processIdRemoval(redactionLogEntry, manualRemoval); } if (item instanceof ManualForceRedaction manualForceRedact) { @@ -145,11 +145,11 @@ public class LegacyRedactionLogMergeService { } if (item instanceof ManualLegalBasisChange manualLegalBasisChange) { - processManualLegalBasisChange(redactionLogEntry, dossierTemplateId, manualLegalBasisChange); + processManualLegalBasisChange(redactionLogEntry, manualLegalBasisChange); } if (item instanceof ManualResizeRedaction manualResizeRedact) { - processManualResizeRedaction(redactionLogEntry, dossierTemplateId, manualResizeRedact); + processManualResizeRedaction(redactionLogEntry, manualResizeRedact); } }); @@ -175,7 +175,6 @@ public class LegacyRedactionLogMergeService { manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", recategorized by manual override"); } else if (imageRecategorization.getStatus().equals(AnnotationStatus.REQUESTED)) { manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to recategorize"); - redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted())); } if (manualOverrideReason != null) { @@ -200,22 +199,19 @@ public class LegacyRedactionLogMergeService { } - private void processIdRemoval(RedactionLogEntry redactionLogEntry,String dossierTemplateId, IdRemoval manualRemoval) { + private void processIdRemoval(RedactionLogEntry redactionLogEntry, IdRemoval manualRemoval) { boolean isApprovedRedaction = manualRemoval.getStatus().equals(AnnotationStatus.APPROVED); if (isApprovedRedaction && manualRemoval.isRemoveFromDictionary() && isBasedOnDictionaryOnly(redactionLogEntry)) { log.debug("Skipping merge for dictionary-modifying entry"); } else { - String redactionLogEntryType = redactionLogEntry.getType(); String manualOverrideReason = null; if (isApprovedRedaction) { redactionLogEntry.setRedacted(false); manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", removed by manual override"); - redactionLogEntry.setColor(getColor(redactionLogEntryType, dossierTemplateId, false, redactionLogEntry.isRedacted())); redactionLogEntry.setHint(false); } else if (manualRemoval.getStatus().equals(AnnotationStatus.REQUESTED)) { manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to remove"); - redactionLogEntry.setColor(getColor(redactionLogEntryType, dossierTemplateId, true, redactionLogEntry.isRedacted())); } if (manualOverrideReason != null) { @@ -246,12 +242,10 @@ public class LegacyRedactionLogMergeService { } else { redactionLogEntry.setRedacted(true); } - redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted())); manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", forced by manual override"); redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis()); } else if (manualForceRedact.getStatus().equals(AnnotationStatus.REQUESTED)) { manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to force " + (dictionaryIsHint ? "hint" : "redact")); - redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted())); redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis()); } @@ -265,7 +259,7 @@ public class LegacyRedactionLogMergeService { } - private void processManualLegalBasisChange(RedactionLogEntry redactionLogEntry, String dossierTemplateId, ManualLegalBasisChange manualLegalBasisChange) { + private void processManualLegalBasisChange(RedactionLogEntry redactionLogEntry, ManualLegalBasisChange manualLegalBasisChange) { String manualOverrideReason = null; if (manualLegalBasisChange.getStatus().equals(AnnotationStatus.APPROVED)) { @@ -280,7 +274,6 @@ public class LegacyRedactionLogMergeService { } } else if (manualLegalBasisChange.getStatus().equals(AnnotationStatus.REQUESTED)) { manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis change requested"); - redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted())); } if (manualOverrideReason != null) { @@ -299,11 +292,10 @@ public class LegacyRedactionLogMergeService { } - private void processManualResizeRedaction(RedactionLogEntry redactionLogEntry, String dossierTemplateId, ManualResizeRedaction manualResizeRedact) { + private void processManualResizeRedaction(RedactionLogEntry redactionLogEntry, ManualResizeRedaction manualResizeRedact) { String manualOverrideReason = null; if (manualResizeRedact.getStatus().equals(AnnotationStatus.APPROVED)) { - redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted())); redactionLogEntry.setPositions(convertPositions(manualResizeRedact.getPositions())); if (!"signature".equalsIgnoreCase(redactionLogEntry.getType()) && !"logo".equalsIgnoreCase(redactionLogEntry.getType())) { redactionLogEntry.setValue(manualResizeRedact.getValue()); @@ -316,7 +308,6 @@ public class LegacyRedactionLogMergeService { manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", resized by manual override"); } else if (manualResizeRedact.getStatus().equals(AnnotationStatus.REQUESTED)) { manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to resize redact"); - redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted())); redactionLogEntry.setPositions(convertPositions(manualResizeRedact.getPositions())); // This is for backwards compatibility, now the text after/before is calculated during reanalysis because we need to find dict entries on positions where entries are resized to smaller. @@ -332,14 +323,14 @@ public class LegacyRedactionLogMergeService { } - public List addManualAddEntries(Set manualAdds, String dossierTemplateId, int analysisNumber) { + public List addManualAddEntries(Set manualAdds, int analysisNumber) { List redactionLogEntries = new ArrayList<>(); for (ManualRedactionEntry manualRedactionEntry : manualAdds) { if (shouldCreateManualEntry(manualRedactionEntry)) { - RedactionLogEntry redactionLogEntry = createRedactionLogEntry(manualRedactionEntry, manualRedactionEntry.getAnnotationId(), dossierTemplateId, analysisNumber); + RedactionLogEntry redactionLogEntry = createRedactionLogEntry(manualRedactionEntry, manualRedactionEntry.getAnnotationId(), analysisNumber); redactionLogEntry.setPositions(convertPositions(manualRedactionEntry.getPositions())); redactionLogEntry.setTextBefore(manualRedactionEntry.getTextBefore()); redactionLogEntry.setTextAfter(manualRedactionEntry.getTextAfter()); @@ -367,7 +358,7 @@ public class LegacyRedactionLogMergeService { } - private RedactionLogEntry createRedactionLogEntry(ManualRedactionEntry manualRedactionEntry, String id, String dossierTemplateId, int analysisNumber) { + private RedactionLogEntry createRedactionLogEntry(ManualRedactionEntry manualRedactionEntry, String id, int analysisNumber) { var addToDictionary = manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary(); @@ -377,7 +368,6 @@ public class LegacyRedactionLogMergeService { return RedactionLogEntry.builder() .id(id) - .color(getColorForManualAdd(manualRedactionEntry.getType(), dossierTemplateId, manualRedactionEntry.getStatus())) .reason(manualRedactionEntry.getReason()) .isDictionaryEntry(manualRedactionEntry.isAddToDictionary()) .isDossierDictionaryEntry(manualRedactionEntry.isAddToDossierDictionary()) @@ -396,30 +386,6 @@ public class LegacyRedactionLogMergeService { } - private float[] getColor(String type, String dossierTemplateId, boolean isRedaction, boolean skipped) { - - if (skipped || !isRedaction && !isHint(dossierTemplateId, type)) { - return dictionaryService.getNotRedactedColor(dossierTemplateId); - } - return dictionaryService.getColor(type, dossierTemplateId); - } - - - private float[] getColorForManualAdd(String type, String dossierTemplateId, AnnotationStatus status) { - - if (status.equals(AnnotationStatus.REQUESTED) || status.equals(AnnotationStatus.DECLINED)) { - return dictionaryService.getNotRedactedColor(dossierTemplateId); - } - return dictionaryService.getColor(type, dossierTemplateId); - } - - - boolean isHint(String dossierTemplateId, String type) { - - return dictionaryService.isHint(type, dossierTemplateId); - } - - @Data @AllArgsConstructor private static class ManualRedactionWrapper implements Comparable { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigrationEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigrationEntity.java index 444047b2..0b18cad2 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigrationEntity.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigrationEntity.java @@ -20,6 +20,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlo import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity; +import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite; import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage; import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image; @@ -51,7 +52,7 @@ public final class MigrationEntity { String ruleIdentifier = "OLD." + redactionLogEntry.getMatchedRule() + ".0"; List rectangleWithPages = redactionLogEntry.getPositions().stream().map(RectangleWithPage::fromRedactionLogRectangle).toList(); - + EntityType entityType = getEntityType(redactionLogEntry); return ManualEntity.builder() .id(redactionLogEntry.getId()) .value(redactionLogEntry.getValue()) @@ -61,11 +62,12 @@ public final class MigrationEntity { .legalBasis(redactionLogEntry.getLegalBasis()) .type(redactionLogEntry.getType()) .section(redactionLogEntry.getSection()) - .entityType(getEntityType(redactionLogEntry)) + .entityType(entityType) .applied(redactionLogEntry.isRedacted()) .isDictionaryEntry(redactionLogEntry.isDictionaryEntry()) .isDossierDictionaryEntry(redactionLogEntry.isDossierDictionaryEntry()) .rectangle(redactionLogEntry.isRectangle()) + .manualOverwrite(new ManualChangeOverwrite(entityType)) .build(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionLogCrawler.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionLogCrawler.java index add3435b..c5b00869 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionLogCrawler.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionLogCrawler.java @@ -2,13 +2,18 @@ package com.iqser.red.service.redaction.v1.server; import java.io.File; import java.io.FileInputStream; +import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.HashSet; +import java.util.Set; import org.junit.jupiter.api.Test; import com.iqser.red.commons.jackson.ObjectMapperFactory; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog; +import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry; import lombok.SneakyThrows; @@ -19,24 +24,69 @@ public class RedactionLogCrawler { public void findRedactionLogVersion0() { String redactionLogsDir = "/home/kschuettler/Nextcloud/redaction_logs/"; - Files.walk(Path.of(redactionLogsDir)) + long version0Count = Files.walk(Path.of(redactionLogsDir)) + .parallel() + .filter(file -> { + try { + return Files.size(file) > 0; + } catch (IOException e) { + throw new RuntimeException(e); + } + }) .map(Path::toFile) .filter(File::isFile) .filter(file -> file.toString().endsWith("redactionlog.json")) .peek(System.out::println) .map(this::parseRedactionLog) - .filter(log -> log.getAnalysisVersion() == 0) + .filter(LogAndFile::isValid) + .filter(LogAndFile::hasAnyDuplicates) + .peek(LogAndFile::saveToTmp) .count(); + System.out.println(version0Count + " version 0 redaction logs found"); } @SneakyThrows - private RedactionLog parseRedactionLog(File file) { + private LogAndFile parseRedactionLog(File file) { var mapper = ObjectMapperFactory.create(); try (var in = new FileInputStream(file)) { - return mapper.readValue(in.readAllBytes(), RedactionLog.class); + return new LogAndFile(mapper.readValue(in.readAllBytes(), RedactionLog.class), file); } } + + private record LogAndFile(RedactionLog redactionLog, File file) { + + public boolean hasAnyDuplicates() { + + Set uniqueStrings = new HashSet<>(); + + for (String str : redactionLog.getRedactionLogEntry().stream().map(RedactionLogEntry::getId).toList()) { + if (!uniqueStrings.add(str)) { + return true; + } + } + + return false; + } + + + @SneakyThrows + public void saveToTmp() { + + Path original = file.toPath(); + Path tmpFile = Path.of("/tmp/").resolve(Path.of("/home/kschuettler/Nextcloud").relativize(original)); + tmpFile.toFile().mkdirs(); + Files.copy(original, tmpFile, StandardCopyOption.REPLACE_EXISTING); + } + + + public boolean isValid() { + + return redactionLog.getRedactionLogEntry() != null; + } + + } + }