Merge branch 'RED-7384' into 'release/4.244.x'

RED-7384: migration fixes, ignore non-local manual changes in migration

See merge request redactmanager/redaction-service!344

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/RedactionLogToEntityLogMigrationService.java

@@ -72,7 +72,7 @@ public class RedactionLogToEntityLogMigrationService {
                 .collect(new MigratedIdsCollector());
 
         log.info("applying manual changes to migrated entities for file {}", fileId);
-        applyProcessedManualChanges(entitiesToMigrate, manualRedactions);
+        applyLocalProcessedManualChanges(entitiesToMigrate, manualRedactions);
 
         EntityLog entityLog = new EntityLog();
         entityLog.setAnalysisNumber(redactionLog.getAnalysisNumber());
@@ -117,7 +117,7 @@ public class RedactionLogToEntityLogMigrationService {
     }
 
 
-    private void applyProcessedManualChanges(List<MigrationEntity> entitiesToMigrate, ManualRedactions manualRedactions) {
+    private void applyLocalProcessedManualChanges(List<MigrationEntity> entitiesToMigrate, ManualRedactions manualRedactions) {
 
         if (manualRedactions == null) {
             return;
@@ -131,6 +131,7 @@ public class RedactionLogToEntityLogMigrationService {
                                                                                    manualRedactions.getRecategorizations())
                 .flatMap(Collection::stream)
                 .filter(manualChange -> manualChange.getProcessedDate() != null)
+                .filter(BaseAnnotation::isLocal)
                 .collect(Collectors.groupingBy(BaseAnnotation::getAnnotationId));
 
         entitiesToMigrate.forEach(migrationEntity -> migrationEntity.applyManualChanges(manualChangesPerAnnotationId.getOrDefault(migrationEntity.getOldId(),
@@ -151,7 +152,7 @@ public class RedactionLogToEntityLogMigrationService {
 
     private List<MigrationEntity> calculateMigrationEntitiesFromRedactionLog(RedactionLog redactionLog, Document document, String dossierTemplateId, String fileId) {
 
-        List<MigrationEntity> images = getImageBasedMigrationEntities(redactionLog, document, fileId);
+        List<MigrationEntity> images = getImageBasedMigrationEntities(redactionLog, document, fileId, dossierTemplateId);
         List<MigrationEntity> textMigrationEntities = getTextBasedMigrationEntities(redactionLog, document, dossierTemplateId, fileId);
         return Stream.of(textMigrationEntities.stream(), images.stream())
                 .flatMap(Function.identity())
@@ -165,7 +166,7 @@ public class RedactionLogToEntityLogMigrationService {
     }
 
 
-    private List<MigrationEntity> getImageBasedMigrationEntities(RedactionLog redactionLog, Document document, String fileId) {
+    private List<MigrationEntity> getImageBasedMigrationEntities(RedactionLog redactionLog, Document document, String fileId, String dossierTemplateId) {
 
         List<Image> images = document.streamAllImages()
                 .collect(Collectors.toList());
@@ -212,7 +213,7 @@ public class RedactionLogToEntityLogMigrationService {
             } else {
                 closestImage.skip(ruleIdentifier, reason);
             }
-            migrationEntities.add(MigrationEntity.fromRedactionLogImage(redactionLogImage, closestImage, fileId));
+            migrationEntities.add(MigrationEntity.fromRedactionLogImage(redactionLogImage, closestImage, fileId, dictionaryService, dossierTemplateId));
         }
         return migrationEntities;
     }
@@ -259,7 +260,7 @@ public class RedactionLogToEntityLogMigrationService {
                 .stream()
                 .filter(redactionLogEntry -> !redactionLogEntry.isImage())
                 .filter(redactionLogEntry -> isOnExistingPage(redactionLogEntry, document.getNumberOfPages()))
-                .map(entry -> MigrationEntity.fromRedactionLogEntry(entry, dictionaryService.isHint(entry.getType(), dossierTemplateId), fileId))
+                .map(entry -> MigrationEntity.fromRedactionLogEntry(entry, fileId, dictionaryService, dossierTemplateId))
                 .toList();
 
         List<PrecursorEntity> precursorEntities = entitiesToMigrate.stream()

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigrationEntity.java

@@ -20,6 +20,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType;
@@ -32,6 +33,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualCha
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
+import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
 import com.iqser.red.service.redaction.v1.server.service.ManualChangeFactory;
 import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
 
@@ -50,6 +52,8 @@ public final class MigrationEntity {
 
     private final PrecursorEntity precursorEntity;
     private final RedactionLogEntry redactionLogEntry;
+    private final DictionaryService dictionaryService;
+    private final String dossierTemplateId;
     private IEntity migratedEntity;
     private String oldId;
     private String newId;
@@ -59,10 +63,10 @@ public final class MigrationEntity {
     List<BaseAnnotation> manualChanges = new LinkedList<>();
 
 
-    public static MigrationEntity fromRedactionLogEntry(RedactionLogEntry redactionLogEntry, boolean hint, String fileId) {
+    public static MigrationEntity fromRedactionLogEntry(RedactionLogEntry redactionLogEntry, String fileId, DictionaryService dictionaryService, String dossierTemplateId) {
 
+        boolean hint = dictionaryService.isHint(redactionLogEntry.getType(), dossierTemplateId);
         PrecursorEntity precursorEntity = createPrecursorEntity(redactionLogEntry, hint);
-
         if (precursorEntity.getEntityType().equals(EntityType.HINT) && !redactionLogEntry.isHint() && !redactionLogEntry.isRedacted()) {
             precursorEntity.ignore(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
         } else if (redactionLogEntry.lastChangeIsRemoved()) {
@@ -77,13 +81,32 @@ public final class MigrationEntity {
             precursorEntity.skip(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
         }
 
-        return MigrationEntity.builder().precursorEntity(precursorEntity).redactionLogEntry(redactionLogEntry).oldId(redactionLogEntry.getId()).fileId(fileId).build();
+        return MigrationEntity.builder()
+                .precursorEntity(precursorEntity)
+                .redactionLogEntry(redactionLogEntry)
+                .oldId(redactionLogEntry.getId())
+                .fileId(fileId)
+                .dictionaryService(dictionaryService)
+                .dossierTemplateId(dossierTemplateId)
+                .build();
     }
 
 
-    public static MigrationEntity fromRedactionLogImage(RedactionLogEntry redactionLogImage, Image image, String fileId) {
+    public static MigrationEntity fromRedactionLogImage(RedactionLogEntry redactionLogImage,
+                                                        Image image,
+                                                        String fileId,
+                                                        DictionaryService dictionaryService,
+                                                        String dossierTemplateId) {
 
-        return MigrationEntity.builder().redactionLogEntry(redactionLogImage).migratedEntity(image).oldId(redactionLogImage.getId()).newId(image.getId()).fileId(fileId).build();
+        return MigrationEntity.builder()
+                .redactionLogEntry(redactionLogImage)
+                .migratedEntity(image)
+                .oldId(redactionLogImage.getId())
+                .newId(image.getId())
+                .fileId(fileId)
+                .dictionaryService(dictionaryService)
+                .dossierTemplateId(dossierTemplateId)
+                .build();
     }
 
 
@@ -160,18 +183,6 @@ public final class MigrationEntity {
     }
 
 
-    private static EntryType getEntryType(EntityType entityType) {
-
-        return switch (entityType) {
-            case ENTITY -> EntryType.ENTITY;
-            case HINT -> EntryType.HINT;
-            case FALSE_POSITIVE -> EntryType.FALSE_POSITIVE;
-            case RECOMMENDATION -> EntryType.RECOMMENDATION;
-            case FALSE_RECOMMENDATION -> EntryType.FALSE_RECOMMENDATION;
-        };
-    }
-
-
     public EntityLogEntry toEntityLogEntry(Map<String, String> oldToNewIdMapping) {
 
         EntityLogEntry entityLogEntry;
@@ -246,7 +257,7 @@ public final class MigrationEntity {
                 .textBefore(redactionLogEntry.getTextBefore())
                 .imageHasTransparency(image.isTransparent())
                 .state(buildEntryState(image))
-                .entryType(redactionLogEntry.isHint() ? EntryType.IMAGE_HINT : EntryType.IMAGE)
+                .entryType(dictionaryService.isHint(imageType, dossierTemplateId) ? EntryType.IMAGE_HINT : EntryType.IMAGE)
                 .build();
 
     }
@@ -357,6 +368,9 @@ public final class MigrationEntity {
             if (manualChange instanceof ManualResizeRedaction manualResizeRedaction && migratedEntity instanceof TextEntity textEntity) {
                 manualResizeRedaction.setAnnotationId(newId);
                 manualChangesApplicationService.resize(textEntity, manualResizeRedaction);
+            } else if (manualChange instanceof ManualRecategorization manualRecategorization && migratedEntity instanceof Image image) {
+                image.setImageType(ImageType.fromString(manualRecategorization.getType()));
+                migratedEntity.getManualOverwrite().addChange(manualChange);
             } else {
                 migratedEntity.getManualOverwrite().addChange(manualChange);
             }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java

@@ -13,7 +13,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
 
 import io.micrometer.core.annotation.Timed;
 import lombok.AccessLevel;
@@ -39,9 +38,11 @@ public class EntityChangeLogService {
         boolean hasChanges = false;
 
         for (EntityLogEntry entityLogEntry : newEntityLogEntries) {
+
             Optional<EntityLogEntry> optionalPreviousEntity = previousEntityLogEntries.stream()
                     .filter(entry -> entry.getId().equals(entityLogEntry.getId()))
                     .findAny();
+
             if (optionalPreviousEntity.isEmpty()) {
                 hasChanges = true;
                 entityLogEntry.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, now));
@@ -49,14 +50,18 @@ public class EntityChangeLogService {
             }
 
             EntityLogEntry previousEntity = optionalPreviousEntity.get();
+
             entityLogEntry.getChanges().addAll(previousEntity.getChanges());
+
             if (!previousEntity.getState().equals(entityLogEntry.getState())) {
                 hasChanges = true;
                 ChangeType changeType = calculateChangeType(entityLogEntry.getState(), previousEntity.getState());
                 entityLogEntry.getChanges().add(new Change(analysisNumber, changeType, now));
             }
         }
+
         addRemovedEntriesAsRemoved(previousEntityLogEntries, newEntityLogEntries, manualRedactions, analysisNumber, now);
+
         return hasChanges;
     }
 
@@ -70,11 +75,16 @@ public class EntityChangeLogService {
         Set<String> existingIds = newEntityLogEntries.stream()
                 .map(EntityLogEntry::getId)
                 .collect(Collectors.toSet());
+
         List<EntityLogEntry> removedEntries = previousEntityLogEntries.stream()
                 .filter(entry -> !existingIds.contains(entry.getId()))
                 .toList();
-        removedEntries.forEach(entry -> entry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, now)));
+
-        removedEntries.forEach(entry -> entry.setState(EntryState.REMOVED));
+        removedEntries.stream()
+                .filter(entry -> !entry.getState().equals(EntryState.REMOVED))
+                .peek(entry -> entry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, now)))
+                .forEach(entry -> entry.setState(EntryState.REMOVED));
+
         newEntityLogEntries.addAll(removedEntries);
     }