From 9859d3d797b63d6cd054d820fac680718cd8a3aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kilian=20Sch=C3=BCttler?= Date: Fri, 6 Sep 2024 14:23:55 +0200 Subject: [PATCH] RED-9850: duplicate key exception in mapper --- .../AnalysisFlagsCalculationService.java | 61 ++++++++++---- .../service/EntityLogMergeService.java | 40 +++++---- .../service/EntityLogMongoWrapperService.java | 2 +- .../ManualRedactionMapper.java | 84 ++++++++++++++----- .../repository/FileRepository.java | 2 +- ...alysisFlagsCalculationMessageReceiver.java | 2 +- 6 files changed, 130 insertions(+), 61 deletions(-) diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/AnalysisFlagsCalculationService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/AnalysisFlagsCalculationService.java index 0c61f37da..4333b1c0d 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/AnalysisFlagsCalculationService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/AnalysisFlagsCalculationService.java @@ -8,6 +8,7 @@ import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ViewedPageEntity; +import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; import com.iqser.red.service.persistence.management.v1.processor.model.websocket.FileEventType; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ViewedPagesPersistenceService; @@ -47,12 +48,20 @@ public class AnalysisFlagsCalculationService { long startTime = System.currentTimeMillis(); - var file = fileStatusPersistenceService.getStatus(fileId); - var entityLog = entityLogService.getEntityLog(dossierId, fileId, true); + FlagCalculationData data; + try { + data = getFlagCalculationData(dossierId, fileId); + } catch (NotFoundException e) { + log.warn("EntityLog does not exist, skipping flag calculation."); + fileStatusPersistenceService.setLastFlagCalculation(fileId, OffsetDateTime.now()); + fileStatusPersistenceService.updateFlags(fileId, false, false, false, false, false, false); + return; + } - var viewedPagesForCurrentAssignee = viewedPagesPersistenceService.findViewedPages(fileId, file.getAssignee()); + fileStatusPersistenceService.setLastFlagCalculation(fileId, OffsetDateTime.now()); - Map viewedPages = viewedPagesForCurrentAssignee.stream() + Map viewedPages = data.viewedPagesForCurrentAssignee() + .stream() .collect(Collectors.toMap(ViewedPageEntity::getPage, ViewedPageEntity::getViewedTime)); boolean hasRedactions = false; @@ -65,7 +74,7 @@ public class AnalysisFlagsCalculationService { OffsetDateTime lastRedactionModification = null; OffsetDateTime lastManualChangeDate = null; - for (EntityLogEntry entry : entityLog.getEntityLogEntry()) { + for (EntityLogEntry entry : data.entityLog().getEntityLogEntry()) { String type = getType(entry.getType()); @@ -112,7 +121,7 @@ public class AnalysisFlagsCalculationService { OffsetDateTime viewedPage = entry.getPositions().isEmpty() ? null : viewedPages.get(entry.getPositions() .get(0).getPageNumber()); - if (file.getWorkflowStatus() != WorkflowStatus.APPROVED && lastChange != null && lastChange.getDateTime() != null && viewedPage != null && // + if (data.file().getWorkflowStatus() != WorkflowStatus.APPROVED && lastChange != null && lastChange.getDateTime() != null && viewedPage != null && // viewedPage.isBefore(lastChange.getDateTime())) { hasUpdates = true; } @@ -123,30 +132,50 @@ public class AnalysisFlagsCalculationService { } - log.info("Flag Calculations for file: {} took: {}ms", fileId, System.currentTimeMillis() - startTime); - if (file.isHasRedactions() == hasRedactions - && file.isHasHints() == hasHints - && file.isHasImages() == hasImages - && file.isHasSuggestions() == hasSuggestions - && file.isHasAnnotationComments() == hasComments - && file.isHasUpdates() == hasUpdates) { + log.debug("Flag Calculations for file: {} took: {}ms", fileId, System.currentTimeMillis() - startTime); + if (data.file().isHasRedactions() == hasRedactions + && data.file().isHasHints() == hasHints + && data.file().isHasImages() == hasImages + && data.file().isHasSuggestions() == hasSuggestions + && data.file().isHasAnnotationComments() == hasComments + && data.file().isHasUpdates() == hasUpdates) { log.info("Nothing Changed for file: {}", fileId); } else { fileStatusPersistenceService.updateFlags(fileId, hasRedactions, hasHints, hasImages, hasSuggestions, hasComments, hasUpdates); } - if (lastRedactionModification != null && (file.getRedactionModificationDate() == null || file.getRedactionModificationDate().isBefore(lastRedactionModification))) { + if (lastRedactionModification != null && (data.file().getRedactionModificationDate() == null || data.file() + .getRedactionModificationDate() + .isBefore(lastRedactionModification))) { fileStatusPersistenceService.setLastRedactionModificationDateForFile(fileId, lastRedactionModification); } - if (lastManualChangeDate != null && (file.getLastManualChangeDate() == null || file.getLastManualChangeDate().isBefore(lastManualChangeDate))) { + if (lastManualChangeDate != null && (data.file().getLastManualChangeDate() == null || data.file().getLastManualChangeDate().isBefore(lastManualChangeDate))) { fileStatusPersistenceService.setLastManualChangeDate(fileId, lastManualChangeDate); } - fileStatusPersistenceService.setLastFlagCalculation(fileId, OffsetDateTime.now()); websocketService.sendFileEvent(dossierId, fileId, FileEventType.UPDATE); } + private FlagCalculationData getFlagCalculationData(String dossierId, String fileId) { + + var file = fileStatusPersistenceService.getStatus(fileId); + var entityLog = entityLogService.getEntityLog(dossierId, fileId, true); + var viewedPagesForCurrentAssignee = viewedPagesPersistenceService.findViewedPages(fileId, file.getAssignee()); + + return new FlagCalculationData(file, entityLog, viewedPagesForCurrentAssignee); + } + + + private record FlagCalculationData( + com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity file, + com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog entityLog, + java.util.List viewedPagesForCurrentAssignee + ) { + + } + + private void addIdsToTrace(String dossierId, String fileId) { if (observationRegistry.getCurrentObservation() != null) { diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogMergeService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogMergeService.java index fe6a59b91..0d8da877c 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogMergeService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogMergeService.java @@ -74,43 +74,41 @@ public class EntityLogMergeService { Map> allManualChanges = groupManualChanges(unprocessedManualRedactions); - List entityLogEntries = new LinkedList<>(entityLog.getEntityLogEntry()); + List mergedEntityLogEntries = merge(unprocessedManualRedactions, entityLog.getEntityLogEntry(), dossier, analysisNumber, allManualChanges); - merge(unprocessedManualRedactions, entityLog.getEntityLogEntry(), dossier, analysisNumber, entityLogEntries, allManualChanges); - - entityLog.setEntityLogEntry(entityLogEntries); + entityLog.setEntityLogEntry(mergedEntityLogEntries); return entityLog; } @Observed(name = "EntityLogMergeService", contextualName = "merge-entity-log-entries") - public List mergeEntityLogEntries(ManualRedactions unprocessedManualRedactions, List entityLogEntryIds, DossierEntity dossier, String fileId) { + public List mergeEntityLogEntries(ManualRedactions unprocessedManualRedactions, List entityLogEntries, DossierEntity dossier, String fileId) { final int analysisNumber = entityLogMongoService.findLatestAnalysisNumber(dossier.getId(), fileId) .orElseThrow(() -> new BadRequestException("Can't load analysis number")); Map> allManualChanges = groupManualChanges(unprocessedManualRedactions); - List entityLogEntries = entityLogMongoService.findEntityLogEntriesByIds(dossier.getId(), fileId, entityLogEntryIds); - - merge(unprocessedManualRedactions, entityLogEntries, dossier, analysisNumber, entityLogEntries, allManualChanges); - - return entityLogEntries; + return merge(unprocessedManualRedactions, entityLogEntries, dossier, analysisNumber, allManualChanges); } - private void merge(ManualRedactions unprocessedManualRedactions, - List entityLog, - DossierEntity dossier, - int analysisNumber, - List entityLogEntries, - Map> allManualChanges) { + private List merge(ManualRedactions unprocessedManualRedactions, + List entityLogEntries, + DossierEntity dossier, + int analysisNumber, + Map> allManualChanges) { - Map addedLocalManualEntries = buildUnprocessedLocalManualRedactions(unprocessedManualRedactions, entityLog, dossier, analysisNumber)// + List mergedEntityLogEntries = new LinkedList<>(entityLogEntries); + + Map addedLocalManualEntries = buildUnprocessedLocalManualRedactions(unprocessedManualRedactions, entityLogEntries, dossier, analysisNumber)// .collect(Collectors.toMap(EntityLogEntry::getId, Function.identity())); - entityLogEntries.addAll(addedLocalManualEntries.values()); - buildPendingDictionaryChanges(unprocessedManualRedactions).forEach(entityLogEntries::add); - processEntityLogEntries(dossier, entityLogEntries, addedLocalManualEntries, analysisNumber, allManualChanges); + mergedEntityLogEntries.addAll(addedLocalManualEntries.values()); + buildPendingDictionaryChanges(unprocessedManualRedactions).forEach(mergedEntityLogEntries::add); + + processEntityLogEntries(dossier, mergedEntityLogEntries, addedLocalManualEntries, analysisNumber, allManualChanges); + + return mergedEntityLogEntries; } @@ -342,7 +340,7 @@ public class EntityLogMergeService { entityLogEntry.setState(EntryState.IGNORED); //special case, only for local add, other local changes and then remove - if (!entityLogEntry.getEngines().isEmpty() && Set.of(Engine.MANUAL).containsAll(entityLogEntry.getEngines())) { + if (entityLogEntry.getEngines().equals(Set.of(Engine.MANUAL))) { entityLogEntry.setState(EntryState.REMOVED); } entityLogEntry.getEngines().add(Engine.MANUAL); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogMongoWrapperService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogMongoWrapperService.java index 9814197cd..9e5822079 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogMongoWrapperService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogMongoWrapperService.java @@ -37,7 +37,7 @@ public class EntityLogMongoWrapperService { if (includeUnprocessed) { DossierEntity dossier = dossierService.getDossierById(dossierId); ManualRedactions unprocessedManualRedactions = manualRedactionProviderService.getManualRedactions(fileId, ManualChangesQueryOptions.unprocessedOnly(), ids); - entityLogEntries = entityLogMergeService.mergeEntityLogEntries(unprocessedManualRedactions, entityLogEntries.stream().map(EntityLogEntry::getId).toList(), dossier, fileId); + entityLogEntries = entityLogMergeService.mergeEntityLogEntries(unprocessedManualRedactions, entityLogEntries, dossier, fileId); } return entityLogEntries; } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionMapper.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionMapper.java index 95b30f0a3..c357560ec 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionMapper.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionMapper.java @@ -8,19 +8,17 @@ import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.function.Consumer; -import java.util.function.Function; import java.util.stream.Collectors; import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualRecategorizationEntity; import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException; -import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; import com.iqser.red.service.persistence.management.v1.processor.service.EntityLogMongoWrapperService; import com.iqser.red.service.persistence.service.v1.api.shared.model.RequestEntryPair; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine; -import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AddRedactionRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ForceRedactionRequest; @@ -95,15 +93,24 @@ public class ManualRedactionMapper { .toList(), includeUnprocessed); - Map iddToEntityLogEntryMap = entityLogEntries.stream() - .collect(Collectors.toMap(EntityLogEntry::getId, Function.identity())); + Map> idsToEntityLogEntryMap = entityLogEntries.stream() + .collect(Collectors.groupingBy(EntityLogEntry::getId)); List> requests = new ArrayList<>(); for (var removeRedactionRequest : removeRedactionRequests) { - EntityLogEntry entityLogEntry = iddToEntityLogEntryMap.get(removeRedactionRequest.getAnnotationId()); + if (!idsToEntityLogEntryMap.containsKey(removeRedactionRequest.getAnnotationId())) { + continue; + } + + List entityLogEntriesWithId = idsToEntityLogEntryMap.get(removeRedactionRequest.getAnnotationId()); + + for (EntityLogEntry entityLogEntry : entityLogEntriesWithId) { + + if (invalidDictionaryRequest(removeRedactionRequest, entityLogEntry) || invalidLocalRequest(removeRedactionRequest, entityLogEntry)) { + continue; + } - if (entityLogEntry != null) { var request = RemoveRedactionRequest.builder() .annotationId(removeRedactionRequest.getAnnotationId()) .user(KeycloakSecurity.getUserId()) @@ -127,6 +134,24 @@ public class ManualRedactionMapper { } + private static boolean invalidLocalRequest(RemoveRedactionRequestModel removeRedactionRequest, EntityLogEntry entityLogEntry) { + + return !isDictionaryRequest(removeRedactionRequest) && entityLogEntry.getState().equals(EntryState.REMOVED); + } + + + private static boolean isDictionaryRequest(RemoveRedactionRequestModel removeRedactionRequest) { + + return removeRedactionRequest.isRemoveFromDictionary() || removeRedactionRequest.isRemoveFromAllDossiers(); + } + + + private static boolean invalidDictionaryRequest(RemoveRedactionRequestModel removeRedactionRequest, EntityLogEntry entityLogEntry) { + + return isDictionaryRequest(removeRedactionRequest) && entityLogEntry.getState().equals(EntryState.PENDING); + } + + public List> toForceRedactionRequestList(String dossierId, String fileId, Set forceRedactionRequests, @@ -199,16 +224,25 @@ public class ManualRedactionMapper { .toList(), includeUnprocessed); - Map annotationIdToEntityLogEntryMap = entityLogEntries.stream() - .collect(Collectors.toMap(EntityLogEntry::getId, Function.identity())); + Map> annotationIdToEntityLogEntryMap = entityLogEntries.stream() + .collect(Collectors.groupingBy(EntityLogEntry::getId)); List> requests = new ArrayList<>(); for (RecategorizationRequestModel recategorizationRequest : recategorizationRequests) { - EntityLogEntry entityLogEntry = annotationIdToEntityLogEntryMap.get(recategorizationRequest.getAnnotationId()); + List entityLogEntriesById = annotationIdToEntityLogEntryMap.get(recategorizationRequest.getAnnotationId()); + + if (entityLogEntriesById == null) { + continue; + } + + for (EntityLogEntry entityLogEntry : entityLogEntriesById) { + + if (invalidDictionaryRequest(recategorizationRequest, entityLogEntry) || invalidLocalRequest(recategorizationRequest, entityLogEntry)) { + continue; + } - if (entityLogEntry != null) { String changedValue; String changedTypeId; @@ -259,6 +293,24 @@ public class ManualRedactionMapper { } + private static boolean invalidLocalRequest(RecategorizationRequestModel removeRedactionRequest, EntityLogEntry entityLogEntry) { + + return !isDictionaryRequest(removeRedactionRequest) && entityLogEntry.getState().equals(EntryState.REMOVED); + } + + + private static boolean isDictionaryRequest(RecategorizationRequestModel removeRedactionRequest) { + + return removeRedactionRequest.isAddToDictionary() || removeRedactionRequest.isAddToAllDossiers(); + } + + + private static boolean invalidDictionaryRequest(RecategorizationRequestModel removeRedactionRequest, EntityLogEntry entityLogEntry) { + + return isDictionaryRequest(removeRedactionRequest) && entityLogEntry.getState().equals(EntryState.PENDING); + } + + private void checkSectionLength(String changedSection) { if (changedSection == null) { @@ -341,16 +393,6 @@ public class ManualRedactionMapper { } - private static EntityLogEntry getEntityLogEntry(EntityLog entityLog, String annotationId) { - - return entityLog.getEntityLogEntry() - .stream() - .filter(entry -> entry.getId().equals(annotationId)) - .findFirst() - .orElseThrow(() -> new NotFoundException("Annotation does not exist in entity log.")); - } - - public static DictionaryEntryType getDictionaryEntryType(EntityLogEntry entityLogEntry) { if (entityLogEntry.getEntryType().equals(EntryType.FALSE_RECOMMENDATION)) { diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java index 902009138..611f84b84 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java @@ -382,7 +382,7 @@ public interface FileRepository extends JpaRepository { select distinct f.id, f.dossierId from FileEntity f left join ViewedPageEntity v on f.id = v.file.id and v.id.userId = f.assignee - where f.deleted is NULL and f.hardDeletedTime is NULL + where f.deleted is NULL and f.hardDeletedTime is NULL and f.processingStatus = 'PROCESSED' and ((f.lastFlagCalculation is NULL and f.lastProcessed is not NULL) or f.lastManualChangeDate > f.lastFlagCalculation or f.lastProcessed > f.lastFlagCalculation or f.lastFlagCalculation < v.viewedTime) """) List getFileIdentifiersWhereAnalysisFlagCalculationIsRequired(); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/AnalysisFlagsCalculationMessageReceiver.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/AnalysisFlagsCalculationMessageReceiver.java index 8ed2d5eca..f72657ba0 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/AnalysisFlagsCalculationMessageReceiver.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/AnalysisFlagsCalculationMessageReceiver.java @@ -32,7 +32,7 @@ public class AnalysisFlagsCalculationMessageReceiver { public void receiveAnalysisFlagCalculationMessage(Message message) { var analysisFlagCalculationMessage = mapper.readValue(message.getBody(), AnalysisFlagCalculationMessage.class); - log.info("Calculating Flags for fileId {} and dossierId {}", analysisFlagCalculationMessage.getFileId(), analysisFlagCalculationMessage.getDossierId()); + log.debug("Calculating Flags for fileId {} and dossierId {}", analysisFlagCalculationMessage.getFileId(), analysisFlagCalculationMessage.getDossierId()); analysisFlagsCalculationService.calculateFlags(analysisFlagCalculationMessage.getDossierId(), analysisFlagCalculationMessage.getFileId()); }