RED-7834: fixes for migration

* remove color
* fix manualChangeOverwrite

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/LegacyRedactionLogMergeService.java

@@ -48,7 +48,7 @@ public class LegacyRedactionLogMergeService {
         log.info("Merging Redaction log with manual redactions");
         if (manualRedactions != null) {
 
-            var manualRedactionLogEntries = addManualAddEntries(manualRedactions.getEntriesToAdd(), dossierTemplateId, redactionLog.getAnalysisNumber());
+            var manualRedactionLogEntries = addManualAddEntries(manualRedactions.getEntriesToAdd(), redactionLog.getAnalysisNumber());
 
             redactionLog.getRedactionLogEntry().addAll(manualRedactionLogEntries);
 
@@ -137,7 +137,7 @@ public class LegacyRedactionLogMergeService {
             }
 
             if (item instanceof IdRemoval manualRemoval) {
-                processIdRemoval(redactionLogEntry, dossierTemplateId, manualRemoval);
+                processIdRemoval(redactionLogEntry, manualRemoval);
             }
 
             if (item instanceof ManualForceRedaction manualForceRedact) {
@@ -145,11 +145,11 @@ public class LegacyRedactionLogMergeService {
             }
 
             if (item instanceof ManualLegalBasisChange manualLegalBasisChange) {
-                processManualLegalBasisChange(redactionLogEntry, dossierTemplateId, manualLegalBasisChange);
+                processManualLegalBasisChange(redactionLogEntry, manualLegalBasisChange);
             }
 
             if (item instanceof ManualResizeRedaction manualResizeRedact) {
-                processManualResizeRedaction(redactionLogEntry, dossierTemplateId, manualResizeRedact);
+                processManualResizeRedaction(redactionLogEntry, manualResizeRedact);
             }
 
         });
@@ -175,7 +175,6 @@ public class LegacyRedactionLogMergeService {
             manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", recategorized by manual override");
         } else if (imageRecategorization.getStatus().equals(AnnotationStatus.REQUESTED)) {
             manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to recategorize");
-            redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted()));
         }
 
         if (manualOverrideReason != null) {
@@ -200,22 +199,19 @@ public class LegacyRedactionLogMergeService {
     }
 
 
-    private void processIdRemoval(RedactionLogEntry redactionLogEntry,String dossierTemplateId, IdRemoval manualRemoval) {
+    private void processIdRemoval(RedactionLogEntry redactionLogEntry, IdRemoval manualRemoval) {
 
         boolean isApprovedRedaction = manualRemoval.getStatus().equals(AnnotationStatus.APPROVED);
         if (isApprovedRedaction && manualRemoval.isRemoveFromDictionary() && isBasedOnDictionaryOnly(redactionLogEntry)) {
             log.debug("Skipping merge for dictionary-modifying entry");
         } else {
-            String redactionLogEntryType = redactionLogEntry.getType();
             String manualOverrideReason = null;
             if (isApprovedRedaction) {
                 redactionLogEntry.setRedacted(false);
                 manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", removed by manual override");
-                redactionLogEntry.setColor(getColor(redactionLogEntryType, dossierTemplateId, false, redactionLogEntry.isRedacted()));
                 redactionLogEntry.setHint(false);
             } else if (manualRemoval.getStatus().equals(AnnotationStatus.REQUESTED)) {
                 manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to remove");
-                redactionLogEntry.setColor(getColor(redactionLogEntryType, dossierTemplateId, true, redactionLogEntry.isRedacted()));
             }
 
             if (manualOverrideReason != null) {
@@ -246,12 +242,10 @@ public class LegacyRedactionLogMergeService {
             } else {
                 redactionLogEntry.setRedacted(true);
             }
-            redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted()));
             manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", forced by manual override");
             redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
         } else if (manualForceRedact.getStatus().equals(AnnotationStatus.REQUESTED)) {
             manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to force " + (dictionaryIsHint ? "hint" : "redact"));
-            redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted()));
             redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
         }
 
@@ -265,7 +259,7 @@ public class LegacyRedactionLogMergeService {
     }
 
 
-    private void processManualLegalBasisChange(RedactionLogEntry redactionLogEntry, String dossierTemplateId, ManualLegalBasisChange manualLegalBasisChange) {
+    private void processManualLegalBasisChange(RedactionLogEntry redactionLogEntry, ManualLegalBasisChange manualLegalBasisChange) {
 
         String manualOverrideReason = null;
         if (manualLegalBasisChange.getStatus().equals(AnnotationStatus.APPROVED)) {
@@ -280,7 +274,6 @@ public class LegacyRedactionLogMergeService {
             }
         } else if (manualLegalBasisChange.getStatus().equals(AnnotationStatus.REQUESTED)) {
             manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis change requested");
-            redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted()));
         }
 
         if (manualOverrideReason != null) {
@@ -299,11 +292,10 @@ public class LegacyRedactionLogMergeService {
     }
 
 
-    private void processManualResizeRedaction(RedactionLogEntry redactionLogEntry, String dossierTemplateId, ManualResizeRedaction manualResizeRedact) {
+    private void processManualResizeRedaction(RedactionLogEntry redactionLogEntry, ManualResizeRedaction manualResizeRedact) {
 
         String manualOverrideReason = null;
         if (manualResizeRedact.getStatus().equals(AnnotationStatus.APPROVED)) {
-            redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted()));
             redactionLogEntry.setPositions(convertPositions(manualResizeRedact.getPositions()));
             if (!"signature".equalsIgnoreCase(redactionLogEntry.getType()) && !"logo".equalsIgnoreCase(redactionLogEntry.getType())) {
                 redactionLogEntry.setValue(manualResizeRedact.getValue());
@@ -316,7 +308,6 @@ public class LegacyRedactionLogMergeService {
             manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", resized by manual override");
         } else if (manualResizeRedact.getStatus().equals(AnnotationStatus.REQUESTED)) {
             manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to resize redact");
-            redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted()));
             redactionLogEntry.setPositions(convertPositions(manualResizeRedact.getPositions()));
 
             // This is for backwards compatibility, now the text after/before is calculated during reanalysis because we need to find dict entries on positions where entries are resized to smaller.
@@ -332,14 +323,14 @@ public class LegacyRedactionLogMergeService {
     }
 
 
-    public List<RedactionLogEntry> addManualAddEntries(Set<ManualRedactionEntry> manualAdds, String dossierTemplateId, int analysisNumber) {
+    public List<RedactionLogEntry> addManualAddEntries(Set<ManualRedactionEntry> manualAdds, int analysisNumber) {
 
         List<RedactionLogEntry> redactionLogEntries = new ArrayList<>();
 
         for (ManualRedactionEntry manualRedactionEntry : manualAdds) {
 
             if (shouldCreateManualEntry(manualRedactionEntry)) {
-                RedactionLogEntry redactionLogEntry = createRedactionLogEntry(manualRedactionEntry, manualRedactionEntry.getAnnotationId(), dossierTemplateId, analysisNumber);
+                RedactionLogEntry redactionLogEntry = createRedactionLogEntry(manualRedactionEntry, manualRedactionEntry.getAnnotationId(), analysisNumber);
                 redactionLogEntry.setPositions(convertPositions(manualRedactionEntry.getPositions()));
                 redactionLogEntry.setTextBefore(manualRedactionEntry.getTextBefore());
                 redactionLogEntry.setTextAfter(manualRedactionEntry.getTextAfter());
@@ -367,7 +358,7 @@ public class LegacyRedactionLogMergeService {
     }
 
 
-    private RedactionLogEntry createRedactionLogEntry(ManualRedactionEntry manualRedactionEntry, String id, String dossierTemplateId, int analysisNumber) {
+    private RedactionLogEntry createRedactionLogEntry(ManualRedactionEntry manualRedactionEntry, String id, int analysisNumber) {
 
         var addToDictionary = manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary();
 
@@ -377,7 +368,6 @@ public class LegacyRedactionLogMergeService {
 
         return RedactionLogEntry.builder()
                 .id(id)
-                .color(getColorForManualAdd(manualRedactionEntry.getType(), dossierTemplateId, manualRedactionEntry.getStatus()))
                 .reason(manualRedactionEntry.getReason())
                 .isDictionaryEntry(manualRedactionEntry.isAddToDictionary())
                 .isDossierDictionaryEntry(manualRedactionEntry.isAddToDossierDictionary())
@@ -396,30 +386,6 @@ public class LegacyRedactionLogMergeService {
     }
 
 
-    private float[] getColor(String type, String dossierTemplateId, boolean isRedaction, boolean skipped) {
-
-        if (skipped || !isRedaction && !isHint(dossierTemplateId, type)) {
-            return dictionaryService.getNotRedactedColor(dossierTemplateId);
-        }
-        return dictionaryService.getColor(type, dossierTemplateId);
-    }
-
-
-    private float[] getColorForManualAdd(String type, String dossierTemplateId, AnnotationStatus status) {
-
-        if (status.equals(AnnotationStatus.REQUESTED) || status.equals(AnnotationStatus.DECLINED)) {
-            return dictionaryService.getNotRedactedColor(dossierTemplateId);
-        }
-        return dictionaryService.getColor(type, dossierTemplateId);
-    }
-
-
-    boolean isHint(String dossierTemplateId, String type) {
-
-        return dictionaryService.isHint(type, dossierTemplateId);
-    }
-
-
     @Data
     @AllArgsConstructor
     private static class ManualRedactionWrapper implements Comparable<ManualRedactionWrapper> {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigrationEntity.java

@@ -20,6 +20,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlo
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
@@ -51,7 +52,7 @@ public final class MigrationEntity {
 
         String ruleIdentifier = "OLD." + redactionLogEntry.getMatchedRule() + ".0";
         List<RectangleWithPage> rectangleWithPages = redactionLogEntry.getPositions().stream().map(RectangleWithPage::fromRedactionLogRectangle).toList();
-
+        EntityType entityType = getEntityType(redactionLogEntry);
         return ManualEntity.builder()
                 .id(redactionLogEntry.getId())
                 .value(redactionLogEntry.getValue())
@@ -61,11 +62,12 @@ public final class MigrationEntity {
                 .legalBasis(redactionLogEntry.getLegalBasis())
                 .type(redactionLogEntry.getType())
                 .section(redactionLogEntry.getSection())
-                .entityType(getEntityType(redactionLogEntry))
+                .entityType(entityType)
                 .applied(redactionLogEntry.isRedacted())
                 .isDictionaryEntry(redactionLogEntry.isDictionaryEntry())
                 .isDossierDictionaryEntry(redactionLogEntry.isDossierDictionaryEntry())
                 .rectangle(redactionLogEntry.isRectangle())
+                .manualOverwrite(new ManualChangeOverwrite(entityType))
                 .build();
     }
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionLogCrawler.java

@@ -2,13 +2,18 @@ package com.iqser.red.service.redaction.v1.server;
 
 import java.io.File;
 import java.io.FileInputStream;
+import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.util.HashSet;
+import java.util.Set;
 
 import org.junit.jupiter.api.Test;
 
 import com.iqser.red.commons.jackson.ObjectMapperFactory;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
 
 import lombok.SneakyThrows;
 
@@ -19,24 +24,69 @@ public class RedactionLogCrawler {
     public void findRedactionLogVersion0() {
 
         String redactionLogsDir = "/home/kschuettler/Nextcloud/redaction_logs/";
-        Files.walk(Path.of(redactionLogsDir))
+        long version0Count = Files.walk(Path.of(redactionLogsDir))
+                .parallel()
+                .filter(file -> {
+                    try {
+                        return Files.size(file) > 0;
+                    } catch (IOException e) {
+                        throw new RuntimeException(e);
+                    }
+                })
                 .map(Path::toFile)
                 .filter(File::isFile)
                 .filter(file -> file.toString().endsWith("redactionlog.json"))
                 .peek(System.out::println)
                 .map(this::parseRedactionLog)
-                .filter(log -> log.getAnalysisVersion() == 0)
+                .filter(LogAndFile::isValid)
+                .filter(LogAndFile::hasAnyDuplicates)
+                .peek(LogAndFile::saveToTmp)
                 .count();
+        System.out.println(version0Count + " version 0 redaction logs found");
     }
 
 
     @SneakyThrows
-    private RedactionLog parseRedactionLog(File file) {
+    private LogAndFile parseRedactionLog(File file) {
 
         var mapper = ObjectMapperFactory.create();
         try (var in = new FileInputStream(file)) {
-            return mapper.readValue(in.readAllBytes(), RedactionLog.class);
+            return new LogAndFile(mapper.readValue(in.readAllBytes(), RedactionLog.class), file);
         }
     }
 
+
+    private record LogAndFile(RedactionLog redactionLog, File file) {
+
+        public boolean hasAnyDuplicates() {
+
+            Set<String> uniqueStrings = new HashSet<>();
+
+            for (String str : redactionLog.getRedactionLogEntry().stream().map(RedactionLogEntry::getId).toList()) {
+                if (!uniqueStrings.add(str)) {
+                    return true;
+                }
+            }
+
+            return false;
+        }
+
+
+        @SneakyThrows
+        public void saveToTmp() {
+
+            Path original = file.toPath();
+            Path tmpFile = Path.of("/tmp/").resolve(Path.of("/home/kschuettler/Nextcloud").relativize(original));
+            tmpFile.toFile().mkdirs();
+            Files.copy(original, tmpFile, StandardCopyOption.REPLACE_EXISTING);
+        }
+
+
+        public boolean isValid() {
+
+            return redactionLog.getRedactionLogEntry() != null;
+        }
+
+    }
+
 }