RED-7384: migration fixes

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogMergeService.java

@@ -14,7 +14,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
-import java.util.Set;
+import java.util.function.Function;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -87,7 +87,12 @@ public class EntityLogMergeService {
 
         List<EntityLogEntry> entityLogEntries = new LinkedList<>(entityLog.getEntityLogEntry());
 
-        buildUnprocessedLocalManualRedactions(unprocessedManualRedactions, entityLog, dossier).forEach(entityLogEntries::add);
+        Map<String, EntityLogEntry> addedLocalManualEntries = buildUnprocessedLocalManualRedactions(unprocessedManualRedactions,
+                                                                                            entityLog,
+                                                                                            dossier).collect(Collectors.toMap(EntityLogEntry::getId, Function.identity()));
+
+        entityLogEntries.addAll(addedLocalManualEntries.values());
+
 
         buildPendingDictionaryChanges(unprocessedManualRedactions).forEach(entityLogEntries::add);
 
@@ -96,6 +101,12 @@ public class EntityLogMergeService {
 
             EntityLogEntry entityLogEntry = entityLogEntries.get(i + numberOfAddedPendingEntries);
 
+            if (isDuplicatedByAny(entityLogEntry, addedLocalManualEntries)) {
+
+                mergeOriginalAndLocallyAddedEntries(addedLocalManualEntries, entityLogEntry, analysisNumber);
+                continue;
+            }
+
             if (allManualChanges.containsKey(entityLogEntry.getId())) {
 
                 List<EntityLogEntry> pendingImageRecategorizations = mergeLocalManualChangesAndReturnNonMergeableAsPending(dossier,
@@ -122,6 +133,18 @@ public class EntityLogMergeService {
     }
 
 
+    private static void mergeOriginalAndLocallyAddedEntries(Map<String, EntityLogEntry> addedLocalManualEntries, EntityLogEntry entityLogEntry, int analysisNumber) {
+
+        EntityLogEntry duplicateManualEntry = addedLocalManualEntries.get(entityLogEntry.getId());
+        duplicateManualEntry.getEngines().addAll(entityLogEntry.getEngines());
+
+        // Mark existing entry REMOVED, will be replaced by the manual one.
+        entityLogEntry.setState(EntryState.REMOVED);
+        entityLogEntry.setReason("Removed due to duplicate locally added manual entity.");
+        entityLogEntry.getChanges().add(Change.builder().type(ChangeType.REMOVED).dateTime(OffsetDateTime.now()).analysisNumber(analysisNumber).build());
+    }
+
+
     private static List<EntityLogEntry> concatLists(List<EntityLogEntry> pendingDictionaryEntries, List<EntityLogEntry> pendingImageRecategorizations) {
 
         return Stream.of(pendingDictionaryEntries, pendingImageRecategorizations)
@@ -235,8 +258,7 @@ public class EntityLogMergeService {
                                   .build());
 
         List<Change> changes = new ArrayList<>();
-        changes.add(Change.builder().analysisNumber(entityLog.getAnalysisNumber()).dateTime(manualRedactionEntry.getRequestDate()).type(ChangeType.ADDED)
+        changes.add(Change.builder().analysisNumber(entityLog.getAnalysisNumber()).dateTime(manualRedactionEntry.getRequestDate()).type(ChangeType.ADDED).build());
-                            .build());
 
         boolean isHint = isHint(manualRedactionEntry.getType(), dossier);
 
@@ -284,8 +306,7 @@ public class EntityLogMergeService {
 
         existingEntry.setState(EntryState.REMOVED);
         List<Change> falsePositiveChanges = new ArrayList<>();
-        falsePositiveChanges.add(Change.builder().analysisNumber(entityLog.getAnalysisNumber()).dateTime(OffsetDateTime.now()).type(ChangeType.REMOVED)
+        falsePositiveChanges.add(Change.builder().analysisNumber(entityLog.getAnalysisNumber()).dateTime(OffsetDateTime.now()).type(ChangeType.REMOVED).build());
-                                         .build());
         if (existingEntry.getChanges() != null && !existingEntry.getChanges().isEmpty()) {
             existingEntry.getChanges().addAll(falsePositiveChanges);
         } else {
@@ -307,7 +328,6 @@ public class EntityLogMergeService {
                              .userId(idRemoval.getUser())
                              .propertyChanges(Collections.emptyMap())
                              .build());
-
     }
 
 
@@ -328,17 +348,19 @@ public class EntityLogMergeService {
             manualChange.propertyChanges(Map.of("value", manualResizeRedaction.getValue()));
         }
         entityLogEntry.getManualChanges().add(manualChange.build());
+    }
 
-        if ((entityLogEntry.isDictionaryEntry() || entityLogEntry.isDossierDictionaryEntry())
+
-            && !manualResizeRedaction.getUpdateDictionary()
+    private boolean isDuplicatedByAny(EntityLogEntry entry, Map<String, EntityLogEntry> addedLocalManualEntryIds) {
-            && !manualResizeRedaction.isAddToAllDossiers()) {
+
-            entityLogEntry.setState(EntryState.REMOVED);
+        return !entry.getEngines().contains(Engine.MANUAL) && addedLocalManualEntryIds.containsKey(entry.getId()) && !addedLocalManualEntryIds.get(entry.getId()).equals(entry);
-        }
     }
 
 
     @Deprecated(forRemoval = true)
-    private void mergeLegalBasisChange(ManualLegalBasisChange manualLegalBasisChange, EntityLogEntry entityLogEntry, int analysisNumber) {
+    private void mergeLegalBasisChange(ManualLegalBasisChange manualLegalBasisChange,
+                                       EntityLogEntry entityLogEntry,
+                                       int analysisNumber) {
 
         entityLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis());
         entityLogEntry.setSection(manualLegalBasisChange.getSection());
@@ -354,10 +376,6 @@ public class EntityLogMergeService {
                              .propertyChanges(propertyChanges)
                              .userId(manualLegalBasisChange.getUser())
                              .build());
-
-        if (entityLogEntry.isDictionaryEntry() || entityLogEntry.isDossierDictionaryEntry()) {
-            entityLogEntry.setState(EntryState.REMOVED);
-        }
     }
 
 
@@ -378,7 +396,10 @@ public class EntityLogMergeService {
     }
 
 
-    private EntityLogEntry mergeRecategorization(ManualRecategorization recategorization, EntityLogEntry entityLogEntry, DossierEntity dossier, int analysisNumber) {
+    private EntityLogEntry mergeRecategorization(ManualRecategorization recategorization,
+                                                 EntityLogEntry entityLogEntry,
+                                                 DossierEntity dossier,
+                                                 int analysisNumber) {
 
         if (entityLogEntry.getEntryType().equals(EntryType.IMAGE) || entityLogEntry.getEntryType().equals(EntryType.IMAGE_HINT)) {
             return pendingDictionaryEntryFactory.buildPendingImageRecategorizationEntry(recategorization, entityLogEntry);
@@ -420,10 +441,6 @@ public class EntityLogMergeService {
                              .userId(recategorization.getUser())
                              .propertyChanges(getPropertyChanges(recategorization))
                              .build());
-
-        if ((entityLogEntry.isDictionaryEntry() || entityLogEntry.isDossierDictionaryEntry()) && !recategorization.isAddToDictionary() && !recategorization.isAddToAllDossiers()) {
-            entityLogEntry.setState(EntryState.REMOVED);
-        }
         return null;
     }
 
@@ -462,10 +479,6 @@ public class EntityLogMergeService {
             forceRedactManualChange.propertyChanges(Map.of("legalBasis", forceRedaction.getLegalBasis()));
         }
         entityLogEntry.getManualChanges().add(forceRedactManualChange.build());
-
-        if (entityLogEntry.isDictionaryEntry() || entityLogEntry.isDossierDictionaryEntry()) {
-            entityLogEntry.setState(EntryState.REMOVED);
-        }
     }
 
 
@@ -482,11 +495,9 @@ public class EntityLogMergeService {
     private void addChanges(List<Change> changes, ChangeType changeType, int analysisNumber, OffsetDateTime offsetDateTime) {
 
         if (!changes.isEmpty()) {
-            changes.add(Change.builder().analysisNumber(analysisNumber + 1).dateTime(offsetDateTime).type(changeType)
+            changes.add(Change.builder().analysisNumber(analysisNumber + 1).dateTime(offsetDateTime).type(changeType).build());
-                                .build());
         } else {
-            changes.add(Change.builder().analysisNumber(analysisNumber).dateTime(OffsetDateTime.now()).type(changeType)
+            changes.add(Change.builder().analysisNumber(analysisNumber).dateTime(OffsetDateTime.now()).type(changeType).build());
-                                .build());
         }
     }
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/IndexingService.java

@@ -9,8 +9,6 @@ import org.apache.commons.lang3.tuple.Pair;
 import org.springframework.amqp.rabbit.core.RabbitTemplate;
 import org.springframework.stereotype.Service;
 
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration;
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
@@ -25,7 +23,6 @@ import lombok.RequiredArgsConstructor;
 public class IndexingService {
 
     private final RabbitTemplate rabbitTemplate;
-    private final ObjectMapper objectMapper;
     private final DossierService dossierService;
     private final FileStatusPersistenceService fileStatusPersistenceService;
 
@@ -53,7 +50,7 @@ public class IndexingService {
             for (Pair<String, String> reindexDossierId : reindexDossierIds) {
                 List<FileEntity> fileStatuses = fileStatusPersistenceService.getStatusesForDossier(reindexDossierId.getRight());
                 for (FileEntity fileStatus : fileStatuses) {
-                    if (fileStatus.isSoftOrHardDeleted()) {
+                    if (fileStatus.getHardDeletedTime() != null) {
                         continue;
                     }
                     if (fileIds != null && !fileIds.isEmpty() && !fileIds.contains(fileStatus.getId())) {

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionMapper.java

@@ -11,7 +11,6 @@ import java.util.Set;
 
 import org.springframework.stereotype.Service;
 
-import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
 import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
 import com.iqser.red.service.persistence.management.v1.processor.service.EntityLogService;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
@@ -60,7 +59,7 @@ public class ManualRedactionMapper {
     }
 
 
-    public AddRedactionRequest toAddRedactionRequest(String dossierId, String dossierTemplateId, AddRedactionRequestModel addRedactionRequest) {
+    private static AddRedactionRequest toAddRedactionRequest(String dossierId, String dossierTemplateId, AddRedactionRequestModel addRedactionRequest) {
 
         return AddRedactionRequest.builder()
                 .value(StringCleaningUtility.cleanString(addRedactionRequest.getValue()))
@@ -128,7 +127,7 @@ public class ManualRedactionMapper {
                     .comment(forceRedactionRequestModel.getComment())
                     .build();
 
-            if (!entityLogEntry.getEngines().contains(Engine.MANUAL)) {
+            if (!entityLogEntry.getEngines().contains(Engine.MANUAL) && entryIsEntityType(entityLogEntry)) {
                 addManualRedactionEntry(fileId, entityLogEntry);
             }
 
@@ -157,7 +156,7 @@ public class ManualRedactionMapper {
                     .value(legalBasisChangeRequest.getValue())
                     .build();
 
-            if (!entityLogEntry.getEngines().contains(Engine.MANUAL)) {
+            if (!entityLogEntry.getEngines().contains(Engine.MANUAL) && entryIsEntityType(entityLogEntry)) {
                 addManualRedactionEntry(fileId, entityLogEntry);
             }
 
@@ -197,7 +196,10 @@ public class ManualRedactionMapper {
                     .section(recategorizationRequest.getSection())
                     .build();
 
-            if (!entityLogEntry.getEngines().contains(Engine.MANUAL) && !recategorizationRequest.isAddToAllDossiers() && !recategorizationRequest.isAddToDictionary()) {
+            if (!entityLogEntry.getEngines().contains(Engine.MANUAL)
+                && !recategorizationRequest.isAddToAllDossiers()
+                && !recategorizationRequest.isAddToDictionary()
+                && entryIsEntityType(entityLogEntry)) {
                 addManualRedactionEntry(fileId, entityLogEntry);
             }
 
@@ -229,7 +231,7 @@ public class ManualRedactionMapper {
                     .addToAllDossiers(resizeRedactionRequest.isAddToAllDossiers())
                     .build();
 
-            if (!entityLogEntry.getEngines().contains(Engine.MANUAL) && !request.isAddToAllDossiers() && !request.getUpdateDictionary()) {
+            if (!entityLogEntry.getEngines().contains(Engine.MANUAL) && entryIsEntityType(entityLogEntry) && !request.isAddToAllDossiers() && !request.getUpdateDictionary()) {
                 addManualRedactionEntry(fileId, entityLogEntry);
             }
 
@@ -240,28 +242,6 @@ public class ManualRedactionMapper {
     }
 
 
-    private EntityLogEntry getEntityLogEntry(EntityLog entityLog, String annotationId) {
-
-        return entityLog.getEntityLogEntry()
-                .stream()
-                .filter(entry -> entry.getId().equals(annotationId))
-                .findFirst()
-                .orElseThrow(() -> new NotFoundException("Annotation does not exist in entity log."));
-    }
-
-
-    private DictionaryEntryType getDictionaryEntryType(EntityLogEntry entityLogEntry) {
-
-        if (entityLogEntry.getEntryType().equals(EntryType.RECOMMENDATION) && entityLogEntry.getEntryType().equals(EntryType.FALSE_POSITIVE)) {
-            return DictionaryEntryType.FALSE_RECOMMENDATION;
-        } else if (entityLogEntry.getEntryType().equals(EntryType.FALSE_POSITIVE)) {
-            return DictionaryEntryType.FALSE_POSITIVE;
-        } else {
-            return DictionaryEntryType.ENTRY;
-        }
-    }
-
-
     private void addManualRedactionEntry(String fileId, EntityLogEntry entityLogEntry) {
 
         ManualRedactionEntry manualRedactionEntry = ManualRedactionEntry.builder()
@@ -278,7 +258,7 @@ public class ManualRedactionMapper {
                 .legalBasis(entityLogEntry.getLegalBasis())
                 .textAfter(entityLogEntry.getTextAfter())
                 .textBefore(entityLogEntry.getTextBefore())
-                .dictionaryEntryType(convertEntryType(entityLogEntry))
+                .dictionaryEntryType(getDictionaryEntryType(entityLogEntry))
                 .fileId(fileId)
                 .requestDate(OffsetDateTime.now())
                 .build();
@@ -287,25 +267,43 @@ public class ManualRedactionMapper {
     }
 
 
-    private List<Rectangle> convertPositions(List<Position> positions) {
+    private static boolean entryIsEntityType(EntityLogEntry entityLogEntry) {
+
+        return entityLogEntry.getEntryType().equals(EntryType.ENTITY) //
+               || entityLogEntry.getEntryType().equals(EntryType.HINT) //
+               || entityLogEntry.getEntryType().equals(EntryType.RECOMMENDATION) //
+               || entityLogEntry.getEntryType().equals(EntryType.FALSE_RECOMMENDATION) //
+               || entityLogEntry.getEntryType().equals(EntryType.FALSE_POSITIVE);
+    }
+
+
+    private static EntityLogEntry getEntityLogEntry(EntityLog entityLog, String annotationId) {
+
+        return entityLog.getEntityLogEntry()
+                .stream()
+                .filter(entry -> entry.getId().equals(annotationId))
+                .findFirst()
+                .orElseThrow(() -> new NotFoundException("Annotation does not exist in entity log."));
+    }
+
+
+    private static DictionaryEntryType getDictionaryEntryType(EntityLogEntry entityLogEntry) {
+
+        if (entityLogEntry.getEntryType().equals(EntryType.FALSE_RECOMMENDATION)) {
+            return DictionaryEntryType.FALSE_RECOMMENDATION;
+        } else if (entityLogEntry.getEntryType().equals(EntryType.FALSE_POSITIVE)) {
+            return DictionaryEntryType.FALSE_POSITIVE;
+        } else {
+            return DictionaryEntryType.ENTRY;
+        }
+    }
+
+
+    private static List<Rectangle> convertPositions(List<Position> positions) {
 
         return positions.stream()
                 .map(rectangle -> new Rectangle(rectangle.x(), rectangle.y(), rectangle.w(), rectangle.h(), rectangle.getPageNumber()))
                 .toList();
     }
 
-
-    private DictionaryEntryType convertEntryType(EntityLogEntry entityLogEntry) {
-
-        if (entityLogEntry.getEntryType().equals(EntryType.FALSE_POSITIVE)) {
-            return DictionaryEntryType.FALSE_POSITIVE;
-        }
-
-        if (entityLogEntry.getEntryType().equals(EntryType.FALSE_RECOMMENDATION)) {
-            return DictionaryEntryType.FALSE_RECOMMENDATION;
-        }
-
-        return DictionaryEntryType.ENTRY;
-    }
-
 }

persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/EntityLogMergeTest.java

@@ -243,14 +243,13 @@ public class EntityLogMergeTest {
         String dossierId = "dossierId";
         String dossierTemplateId = "dossierTemplateId";
 
-        String entryToAddId = UUID.randomUUID().toString();
         String rectangleToAddId = UUID.randomUUID().toString();
         String entryToRemoveId = UUID.randomUUID().toString();
         String entryToResizeId = UUID.randomUUID().toString();
         String entryLegalBasisId = UUID.randomUUID().toString();
         String forceRedactionId = UUID.randomUUID().toString();
 
-        ManualRedactions manualRedactions = provideManualRedactions(entryToAddId, entryToRemoveId, entryToResizeId, entryLegalBasisId, forceRedactionId, fileId, rectangleToAddId);
+        ManualRedactions manualRedactions = provideManualRedactions(entryLegalBasisId, entryToRemoveId, entryToResizeId, entryLegalBasisId, forceRedactionId, fileId, rectangleToAddId);
 
         var entityLog = provideEntityLog(entryToRemoveId, entryToResizeId, entryLegalBasisId, forceRedactionId, true);
 
@@ -272,21 +271,22 @@ public class EntityLogMergeTest {
                 .filter(entityLogEntry1 -> entityLogEntry1.getId().equals(entryToResizeId))
                 .findFirst();
         assertTrue(optionalResizeEntryLogEntry.isPresent());
-        assertEquals(optionalResizeEntryLogEntry.get().getEntryType(), EntryType.ENTITY);
+        assertEquals(EntryType.ENTITY, optionalResizeEntryLogEntry.get().getEntryType());
 
-        var optionalLegalBasisEntryLogEntry = response.getEntityLogEntry()
+        var legalBasisEntries = response.getEntityLogEntry()
                 .stream()
                 .filter(entityLogEntry1 -> entityLogEntry1.getId().equals(entryLegalBasisId))
-                .findFirst();
+                .toList();
-        assertTrue(optionalLegalBasisEntryLogEntry.isPresent());
+        assertEquals(2, legalBasisEntries.size());
-        assertEquals(optionalLegalBasisEntryLogEntry.get().getState(), EntryState.REMOVED);
+        assertEquals(EntryState.REMOVED, legalBasisEntries.get(0).getState());
+        assertEquals(EntryState.APPLIED, legalBasisEntries.get(1).getState());
 
         var optionalForceRedactionEntryLogEntry = response.getEntityLogEntry()
                 .stream()
                 .filter(entityLogEntry1 -> entityLogEntry1.getId().equals(forceRedactionId))
                 .findFirst();
         assertTrue(optionalForceRedactionEntryLogEntry.isPresent());
-        assertEquals(optionalForceRedactionEntryLogEntry.get().getState(), EntryState.REMOVED);
+        assertEquals(EntryState.APPLIED, optionalForceRedactionEntryLogEntry.get().getState());
     }
 
 
@@ -302,7 +302,6 @@ public class EntityLogMergeTest {
         String entryLegalBasisId = UUID.randomUUID().toString();
         String forceRedactionId = UUID.randomUUID().toString();
 
-
         ManualRedactions manualRedactions = new ManualRedactions();
         List<Rectangle> positions = new ArrayList<>();
         positions.add(new Rectangle(2, 2, 2, 2, 1));
@@ -329,10 +328,16 @@ public class EntityLogMergeTest {
 
         EntityLog response = entityLogMergeService.mergeEntityLog(manualRedactions, entityLog, DossierEntity.builder().dossierTemplateId(dossierTemplateId).build());
 
-        var resizedEntry = response.getEntityLogEntry().stream().filter(entityLogEntry1 -> entityLogEntry1.getId().equals(entryToResizeId)).findFirst().get();
+        var resizedEntry = response.getEntityLogEntry()
+                .stream()
+                .filter(entityLogEntry1 -> entityLogEntry1.getId().equals(entryToResizeId))
+                .findFirst()
+                .get();
         int index = response.getEntityLogEntry().indexOf(resizedEntry);
-        assertEquals(response.getEntityLogEntry().get(index + 1).getId(), resizedEntry.getId());
+        assertEquals(response.getEntityLogEntry()
-        assertEquals(response.getEntityLogEntry().get(index + 1).getState(), EntryState.PENDING);
+                             .get(index + 1).getId(), resizedEntry.getId());
+        assertEquals(response.getEntityLogEntry()
+                             .get(index + 1).getState(), EntryState.PENDING);
     }