diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogMergeService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogMergeService.java index bfd679a24..8605daa86 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogMergeService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogMergeService.java @@ -14,6 +14,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -86,7 +87,12 @@ public class EntityLogMergeService { List entityLogEntries = new LinkedList<>(entityLog.getEntityLogEntry()); - buildUnprocessedLocalManualRedactions(unprocessedManualRedactions, entityLog, dossier).forEach(entityLogEntries::add); + Map addedLocalManualEntries = buildUnprocessedLocalManualRedactions(unprocessedManualRedactions, + entityLog, + dossier).collect(Collectors.toMap(EntityLogEntry::getId, Function.identity())); + + entityLogEntries.addAll(addedLocalManualEntries.values()); + buildPendingDictionaryChanges(unprocessedManualRedactions).forEach(entityLogEntries::add); @@ -95,6 +101,12 @@ public class EntityLogMergeService { EntityLogEntry entityLogEntry = entityLogEntries.get(i + numberOfAddedPendingEntries); + if (isDuplicatedByAny(entityLogEntry, addedLocalManualEntries)) { + + mergeOriginalAndLocallyAddedEntries(addedLocalManualEntries, entityLogEntry, analysisNumber); + continue; + } + if (allManualChanges.containsKey(entityLogEntry.getId())) { List pendingImageRecategorizations = mergeLocalManualChangesAndReturnNonMergeableAsPending(dossier, @@ -121,6 +133,18 @@ public class EntityLogMergeService { } + private static void mergeOriginalAndLocallyAddedEntries(Map addedLocalManualEntries, EntityLogEntry entityLogEntry, int analysisNumber) { + + EntityLogEntry duplicateManualEntry = addedLocalManualEntries.get(entityLogEntry.getId()); + duplicateManualEntry.getEngines().addAll(entityLogEntry.getEngines()); + + // Mark existing entry REMOVED, will be replaced by the manual one. + entityLogEntry.setState(EntryState.REMOVED); + entityLogEntry.setReason("Removed due to duplicate locally added manual entity."); + entityLogEntry.getChanges().add(Change.builder().type(ChangeType.REMOVED).dateTime(OffsetDateTime.now()).analysisNumber(analysisNumber).build()); + } + + private static List concatLists(List pendingDictionaryEntries, List pendingImageRecategorizations) { return Stream.of(pendingDictionaryEntries, pendingImageRecategorizations) @@ -234,8 +258,7 @@ public class EntityLogMergeService { .build()); List changes = new ArrayList<>(); - changes.add(Change.builder().analysisNumber(entityLog.getAnalysisNumber()).dateTime(manualRedactionEntry.getRequestDate()).type(ChangeType.ADDED) - .build()); + changes.add(Change.builder().analysisNumber(entityLog.getAnalysisNumber()).dateTime(manualRedactionEntry.getRequestDate()).type(ChangeType.ADDED).build()); boolean isHint = isHint(manualRedactionEntry.getType(), dossier); @@ -283,8 +306,7 @@ public class EntityLogMergeService { existingEntry.setState(EntryState.REMOVED); List falsePositiveChanges = new ArrayList<>(); - falsePositiveChanges.add(Change.builder().analysisNumber(entityLog.getAnalysisNumber()).dateTime(OffsetDateTime.now()).type(ChangeType.REMOVED) - .build()); + falsePositiveChanges.add(Change.builder().analysisNumber(entityLog.getAnalysisNumber()).dateTime(OffsetDateTime.now()).type(ChangeType.REMOVED).build()); if (existingEntry.getChanges() != null && !existingEntry.getChanges().isEmpty()) { existingEntry.getChanges().addAll(falsePositiveChanges); } else { @@ -306,7 +328,6 @@ public class EntityLogMergeService { .userId(idRemoval.getUser()) .propertyChanges(Collections.emptyMap()) .build()); - } @@ -327,16 +348,19 @@ public class EntityLogMergeService { manualChange.propertyChanges(Map.of("value", manualResizeRedaction.getValue())); } entityLogEntry.getManualChanges().add(manualChange.build()); - - if ((entityLogEntry.isDictionaryEntry() || entityLogEntry.isDossierDictionaryEntry()) - && !manualResizeRedaction.getUpdateDictionary() - && !manualResizeRedaction.isAddToAllDossiers()) { - entityLogEntry.setState(EntryState.REMOVED); - } } - private void mergeLegalBasisChange(ManualLegalBasisChange manualLegalBasisChange, EntityLogEntry entityLogEntry, int analysisNumber) { + private boolean isDuplicatedByAny(EntityLogEntry entry, Map addedLocalManualEntryIds) { + + return !entry.getEngines().contains(Engine.MANUAL) && addedLocalManualEntryIds.containsKey(entry.getId()) && !addedLocalManualEntryIds.get(entry.getId()).equals(entry); + } + + + @Deprecated(forRemoval = true) + private void mergeLegalBasisChange(ManualLegalBasisChange manualLegalBasisChange, + EntityLogEntry entityLogEntry, + int analysisNumber) { entityLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis()); entityLogEntry.setSection(manualLegalBasisChange.getSection()); @@ -352,13 +376,10 @@ public class EntityLogMergeService { .propertyChanges(propertyChanges) .userId(manualLegalBasisChange.getUser()) .build()); - - if (entityLogEntry.isDictionaryEntry() || entityLogEntry.isDossierDictionaryEntry()) { - entityLogEntry.setState(EntryState.REMOVED); - } } + @Deprecated(forRemoval = true) private Map getPropertyChanges(ManualLegalBasisChange manualLegalBasisChange) { Map propertyChanges = new HashMap<>(); @@ -375,7 +396,10 @@ public class EntityLogMergeService { } - private EntityLogEntry mergeRecategorization(ManualRecategorization recategorization, EntityLogEntry entityLogEntry, DossierEntity dossier, int analysisNumber) { + private EntityLogEntry mergeRecategorization(ManualRecategorization recategorization, + EntityLogEntry entityLogEntry, + DossierEntity dossier, + int analysisNumber) { if (entityLogEntry.getEntryType().equals(EntryType.IMAGE) || entityLogEntry.getEntryType().equals(EntryType.IMAGE_HINT)) { return pendingDictionaryEntryFactory.buildPendingImageRecategorizationEntry(recategorization, entityLogEntry); @@ -417,10 +441,6 @@ public class EntityLogMergeService { .userId(recategorization.getUser()) .propertyChanges(getPropertyChanges(recategorization)) .build()); - - if ((entityLogEntry.isDictionaryEntry() || entityLogEntry.isDossierDictionaryEntry()) && !recategorization.isAddToDictionary() && !recategorization.isAddToAllDossiers()) { - entityLogEntry.setState(EntryState.REMOVED); - } return null; } @@ -459,10 +479,6 @@ public class EntityLogMergeService { forceRedactManualChange.propertyChanges(Map.of("legalBasis", forceRedaction.getLegalBasis())); } entityLogEntry.getManualChanges().add(forceRedactManualChange.build()); - - if (entityLogEntry.isDictionaryEntry() || entityLogEntry.isDossierDictionaryEntry()) { - entityLogEntry.setState(EntryState.REMOVED); - } } @@ -479,11 +495,9 @@ public class EntityLogMergeService { private void addChanges(List changes, ChangeType changeType, int analysisNumber, OffsetDateTime offsetDateTime) { if (!changes.isEmpty()) { - changes.add(Change.builder().analysisNumber(analysisNumber + 1).dateTime(offsetDateTime).type(changeType) - .build()); + changes.add(Change.builder().analysisNumber(analysisNumber + 1).dateTime(offsetDateTime).type(changeType).build()); } else { - changes.add(Change.builder().analysisNumber(analysisNumber).dateTime(OffsetDateTime.now()).type(changeType) - .build()); + changes.add(Change.builder().analysisNumber(analysisNumber).dateTime(OffsetDateTime.now()).type(changeType).build()); } } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/IndexingService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/IndexingService.java index 3d45f6df2..070fd9049 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/IndexingService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/IndexingService.java @@ -9,8 +9,6 @@ import org.apache.commons.lang3.tuple.Pair; import org.springframework.amqp.rabbit.core.RabbitTemplate; import org.springframework.stereotype.Service; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; @@ -25,7 +23,6 @@ import lombok.RequiredArgsConstructor; public class IndexingService { private final RabbitTemplate rabbitTemplate; - private final ObjectMapper objectMapper; private final DossierService dossierService; private final FileStatusPersistenceService fileStatusPersistenceService; @@ -53,7 +50,7 @@ public class IndexingService { for (Pair reindexDossierId : reindexDossierIds) { List fileStatuses = fileStatusPersistenceService.getStatusesForDossier(reindexDossierId.getRight()); for (FileEntity fileStatus : fileStatuses) { - if (fileStatus.isSoftOrHardDeleted()) { + if (fileStatus.getHardDeletedTime() != null) { continue; } if (fileIds != null && !fileIds.isEmpty() && !fileIds.contains(fileStatus.getId())) { diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionMapper.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionMapper.java index c1ce4e123..695f9c220 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionMapper.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionMapper.java @@ -59,7 +59,7 @@ public class ManualRedactionMapper { } - public AddRedactionRequest toAddRedactionRequest(String dossierId, String dossierTemplateId, AddRedactionRequestModel addRedactionRequest) { + private static AddRedactionRequest toAddRedactionRequest(String dossierId, String dossierTemplateId, AddRedactionRequestModel addRedactionRequest) { return AddRedactionRequest.builder() .value(StringCleaningUtility.cleanString(addRedactionRequest.getValue())) @@ -129,7 +129,7 @@ public class ManualRedactionMapper { .comment(forceRedactionRequestModel.getComment()) .build(); - if (!entityLogEntry.getEngines().contains(Engine.MANUAL)) { + if (!entityLogEntry.getEngines().contains(Engine.MANUAL) && entryIsEntityType(entityLogEntry)) { addManualRedactionEntry(fileId, entityLogEntry); } @@ -158,7 +158,7 @@ public class ManualRedactionMapper { .value(legalBasisChangeRequest.getValue()) .build(); - if (!entityLogEntry.getEngines().contains(Engine.MANUAL)) { + if (!entityLogEntry.getEngines().contains(Engine.MANUAL) && entryIsEntityType(entityLogEntry)) { addManualRedactionEntry(fileId, entityLogEntry); } @@ -198,7 +198,10 @@ public class ManualRedactionMapper { .section(recategorizationRequest.getSection()) .build(); - if (!entityLogEntry.getEngines().contains(Engine.MANUAL) && !recategorizationRequest.isAddToAllDossiers() && !recategorizationRequest.isAddToDictionary()) { + if (!entityLogEntry.getEngines().contains(Engine.MANUAL) + && !recategorizationRequest.isAddToAllDossiers() + && !recategorizationRequest.isAddToDictionary() + && entryIsEntityType(entityLogEntry)) { addManualRedactionEntry(fileId, entityLogEntry); } @@ -230,7 +233,7 @@ public class ManualRedactionMapper { .addToAllDossiers(resizeRedactionRequest.isAddToAllDossiers()) .build(); - if (!entityLogEntry.getEngines().contains(Engine.MANUAL) && !request.isAddToAllDossiers() && !request.getUpdateDictionary()) { + if (!entityLogEntry.getEngines().contains(Engine.MANUAL) && entryIsEntityType(entityLogEntry) && !request.isAddToAllDossiers() && !request.getUpdateDictionary()) { addManualRedactionEntry(fileId, entityLogEntry); } @@ -241,28 +244,6 @@ public class ManualRedactionMapper { } - private EntityLogEntry getEntityLogEntry(EntityLog entityLog, String annotationId) { - - return entityLog.getEntityLogEntry() - .stream() - .filter(entry -> entry.getId().equals(annotationId)) - .findFirst() - .orElseThrow(() -> new NotFoundException("Annotation does not exist in entity log.")); - } - - - private DictionaryEntryType getDictionaryEntryType(EntityLogEntry entityLogEntry) { - - if (entityLogEntry.getEntryType().equals(EntryType.RECOMMENDATION) && entityLogEntry.getEntryType().equals(EntryType.FALSE_POSITIVE)) { - return DictionaryEntryType.FALSE_RECOMMENDATION; - } else if (entityLogEntry.getEntryType().equals(EntryType.FALSE_POSITIVE)) { - return DictionaryEntryType.FALSE_POSITIVE; - } else { - return DictionaryEntryType.ENTRY; - } - } - - private void addManualRedactionEntry(String fileId, EntityLogEntry entityLogEntry) { ManualRedactionEntry manualRedactionEntry = ManualRedactionEntry.builder() @@ -279,7 +260,7 @@ public class ManualRedactionMapper { .legalBasis(entityLogEntry.getLegalBasis()) .textAfter(entityLogEntry.getTextAfter()) .textBefore(entityLogEntry.getTextBefore()) - .dictionaryEntryType(convertEntryType(entityLogEntry)) + .dictionaryEntryType(getDictionaryEntryType(entityLogEntry)) .fileId(fileId) .requestDate(OffsetDateTime.now()) .build(); @@ -288,25 +269,43 @@ public class ManualRedactionMapper { } - private List convertPositions(List positions) { + private static boolean entryIsEntityType(EntityLogEntry entityLogEntry) { + + return entityLogEntry.getEntryType().equals(EntryType.ENTITY) // + || entityLogEntry.getEntryType().equals(EntryType.HINT) // + || entityLogEntry.getEntryType().equals(EntryType.RECOMMENDATION) // + || entityLogEntry.getEntryType().equals(EntryType.FALSE_RECOMMENDATION) // + || entityLogEntry.getEntryType().equals(EntryType.FALSE_POSITIVE); + } + + + private static EntityLogEntry getEntityLogEntry(EntityLog entityLog, String annotationId) { + + return entityLog.getEntityLogEntry() + .stream() + .filter(entry -> entry.getId().equals(annotationId)) + .findFirst() + .orElseThrow(() -> new NotFoundException("Annotation does not exist in entity log.")); + } + + + private static DictionaryEntryType getDictionaryEntryType(EntityLogEntry entityLogEntry) { + + if (entityLogEntry.getEntryType().equals(EntryType.FALSE_RECOMMENDATION)) { + return DictionaryEntryType.FALSE_RECOMMENDATION; + } else if (entityLogEntry.getEntryType().equals(EntryType.FALSE_POSITIVE)) { + return DictionaryEntryType.FALSE_POSITIVE; + } else { + return DictionaryEntryType.ENTRY; + } + } + + + private static List convertPositions(List positions) { return positions.stream() .map(rectangle -> new Rectangle(rectangle.x(), rectangle.y(), rectangle.w(), rectangle.h(), rectangle.getPageNumber())) .toList(); } - - private DictionaryEntryType convertEntryType(EntityLogEntry entityLogEntry) { - - if (entityLogEntry.getEntryType().equals(EntryType.FALSE_POSITIVE)) { - return DictionaryEntryType.FALSE_POSITIVE; - } - - if (entityLogEntry.getEntryType().equals(EntryType.FALSE_RECOMMENDATION)) { - return DictionaryEntryType.FALSE_RECOMMENDATION; - } - - return DictionaryEntryType.ENTRY; - } - } diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/EntityLogMergeTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/EntityLogMergeTest.java index bad993284..10644297c 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/EntityLogMergeTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/EntityLogMergeTest.java @@ -243,14 +243,13 @@ public class EntityLogMergeTest { String dossierId = "dossierId"; String dossierTemplateId = "dossierTemplateId"; - String entryToAddId = UUID.randomUUID().toString(); String rectangleToAddId = UUID.randomUUID().toString(); String entryToRemoveId = UUID.randomUUID().toString(); String entryToResizeId = UUID.randomUUID().toString(); String entryLegalBasisId = UUID.randomUUID().toString(); String forceRedactionId = UUID.randomUUID().toString(); - ManualRedactions manualRedactions = provideManualRedactions(entryToAddId, entryToRemoveId, entryToResizeId, entryLegalBasisId, forceRedactionId, fileId, rectangleToAddId); + ManualRedactions manualRedactions = provideManualRedactions(entryLegalBasisId, entryToRemoveId, entryToResizeId, entryLegalBasisId, forceRedactionId, fileId, rectangleToAddId); var entityLog = provideEntityLog(entryToRemoveId, entryToResizeId, entryLegalBasisId, forceRedactionId, true); @@ -272,21 +271,22 @@ public class EntityLogMergeTest { .filter(entityLogEntry1 -> entityLogEntry1.getId().equals(entryToResizeId)) .findFirst(); assertTrue(optionalResizeEntryLogEntry.isPresent()); - assertEquals(optionalResizeEntryLogEntry.get().getEntryType(), EntryType.ENTITY); + assertEquals(EntryType.ENTITY, optionalResizeEntryLogEntry.get().getEntryType()); - var optionalLegalBasisEntryLogEntry = response.getEntityLogEntry() + var legalBasisEntries = response.getEntityLogEntry() .stream() .filter(entityLogEntry1 -> entityLogEntry1.getId().equals(entryLegalBasisId)) - .findFirst(); - assertTrue(optionalLegalBasisEntryLogEntry.isPresent()); - assertEquals(optionalLegalBasisEntryLogEntry.get().getState(), EntryState.REMOVED); + .toList(); + assertEquals(2, legalBasisEntries.size()); + assertEquals(EntryState.REMOVED, legalBasisEntries.get(0).getState()); + assertEquals(EntryState.APPLIED, legalBasisEntries.get(1).getState()); var optionalForceRedactionEntryLogEntry = response.getEntityLogEntry() .stream() .filter(entityLogEntry1 -> entityLogEntry1.getId().equals(forceRedactionId)) .findFirst(); assertTrue(optionalForceRedactionEntryLogEntry.isPresent()); - assertEquals(optionalForceRedactionEntryLogEntry.get().getState(), EntryState.REMOVED); + assertEquals(EntryState.APPLIED, optionalForceRedactionEntryLogEntry.get().getState()); } @@ -302,7 +302,6 @@ public class EntityLogMergeTest { String entryLegalBasisId = UUID.randomUUID().toString(); String forceRedactionId = UUID.randomUUID().toString(); - ManualRedactions manualRedactions = new ManualRedactions(); List positions = new ArrayList<>(); positions.add(new Rectangle(2, 2, 2, 2, 1)); @@ -329,10 +328,16 @@ public class EntityLogMergeTest { EntityLog response = entityLogMergeService.mergeEntityLog(manualRedactions, entityLog, DossierEntity.builder().dossierTemplateId(dossierTemplateId).build()); - var resizedEntry = response.getEntityLogEntry().stream().filter(entityLogEntry1 -> entityLogEntry1.getId().equals(entryToResizeId)).findFirst().get(); + var resizedEntry = response.getEntityLogEntry() + .stream() + .filter(entityLogEntry1 -> entityLogEntry1.getId().equals(entryToResizeId)) + .findFirst() + .get(); int index = response.getEntityLogEntry().indexOf(resizedEntry); - assertEquals(response.getEntityLogEntry().get(index + 1).getId(), resizedEntry.getId()); - assertEquals(response.getEntityLogEntry().get(index + 1).getState(), EntryState.PENDING); + assertEquals(response.getEntityLogEntry() + .get(index + 1).getId(), resizedEntry.getId()); + assertEquals(response.getEntityLogEntry() + .get(index + 1).getState(), EntryState.PENDING); }