diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/AnalysisFlagsCalculationService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/AnalysisFlagsCalculationService.java index 03f7c54c8..e052daee8 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/AnalysisFlagsCalculationService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/AnalysisFlagsCalculationService.java @@ -9,6 +9,7 @@ import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ViewedPageEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; +import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; import com.iqser.red.service.persistence.management.v1.processor.model.AnalysisFlags; import com.iqser.red.service.persistence.management.v1.processor.model.websocket.FileEventType; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService; @@ -51,23 +52,27 @@ public class AnalysisFlagsCalculationService { long startTime = System.currentTimeMillis(); - var file = fileStatusPersistenceService.getStatus(fileId); - var entityLog = entityLogService.getEntityLog(dossierId, fileId, true); + FlagCalculationData data; + try { + data = getFlagCalculationData(dossierId, fileId); + } catch (NotFoundException e) { + log.warn("EntityLog does not exist, skipping flag calculation."); + fileStatusPersistenceService.setLastFlagCalculation(fileId, OffsetDateTime.now()); + fileStatusPersistenceService.updateFlags(fileId, false, false, false, false, false, false); + return; + } - var viewedPagesForCurrentAssignee = viewedPagesPersistenceService.findViewedPages(fileId, file.getAssignee()); + fileStatusPersistenceService.setLastFlagCalculation(fileId, OffsetDateTime.now()); - Map viewedPages = viewedPagesForCurrentAssignee.stream() - .collect(Collectors.toMap(ViewedPageEntity::getPage, ViewedPageEntity::getViewedTime)); - - AnalysisFlags flags = calculateFlags(entityLog, viewedPages, file); + AnalysisFlags flags = calculateFlags(data); log.debug("Flag Calculations for file: {} took: {}ms", fileId, System.currentTimeMillis() - startTime); - if (file.isHasRedactions() == flags.hasRedactions() - && file.isHasHints() == flags.hasHints() - && file.isHasImages() == flags.hasImages() - && file.isHasSuggestions() == flags.hasSuggestions() - && file.isHasAnnotationComments() == flags.hasComments() - && file.isHasUpdates() == flags.hasUpdates()) { + if (data.file().isHasRedactions() == flags.hasRedactions() + && data.file().isHasHints() == flags.hasHints() + && data.file().isHasImages() == flags.hasImages() + && data.file().isHasSuggestions() == flags.hasSuggestions() + && data.file().isHasAnnotationComments() == flags.hasComments() + && data.file().isHasUpdates() == flags.hasUpdates()) { log.debug("Nothing Changed for file: {}", fileId); } else { fileStatusPersistenceService.updateFlags(fileId, @@ -79,11 +84,14 @@ public class AnalysisFlagsCalculationService { flags.hasUpdates()); } - if (flags.lastRedactionModification() != null && (file.getRedactionModificationDate() == null || file.getRedactionModificationDate() + if (flags.lastRedactionModification() != null && (data.file().getRedactionModificationDate() == null || data.file() + .getRedactionModificationDate() .isBefore(flags.lastRedactionModification()))) { fileStatusPersistenceService.setLastRedactionModificationDateForFile(fileId, flags.lastRedactionModification()); } - if (flags.lastManualChangeDate() != null && (file.getLastManualChangeDate() == null || file.getLastManualChangeDate().isBefore(flags.lastManualChangeDate()))) { + if (flags.lastManualChangeDate() != null && (data.file().getLastManualChangeDate() == null || data.file() + .getLastManualChangeDate() + .isBefore(flags.lastManualChangeDate()))) { fileStatusPersistenceService.setLastManualChangeDate(fileId, flags.lastManualChangeDate()); } @@ -92,7 +100,7 @@ public class AnalysisFlagsCalculationService { } - public AnalysisFlags calculateFlags(EntityLog entityLog, Map viewedPages, FileEntity file) { + public AnalysisFlags calculateFlags(FlagCalculationData data) { boolean hasRedactions = false; boolean hasHints = false; @@ -104,7 +112,7 @@ public class AnalysisFlagsCalculationService { OffsetDateTime lastRedactionModification = null; OffsetDateTime lastManualChangeDate = null; - for (EntityLogEntry entry : entityLog.getEntityLogEntry()) { + for (EntityLogEntry entry : data.entityLog().getEntityLogEntry()) { String type = getType(entry.getType()); @@ -152,10 +160,10 @@ public class AnalysisFlagsCalculationService { hasImages = true; } - OffsetDateTime viewedPage = entry.getPositions().isEmpty() ? null : viewedPages.get(entry.getPositions() - .get(0).getPageNumber()); + OffsetDateTime viewedPage = entry.getPositions().isEmpty() ? null : data.viewedPages.get(entry.getPositions() + .get(0).getPageNumber()); - if (file.getWorkflowStatus() != WorkflowStatus.APPROVED && lastChange != null && lastChange.getDateTime() != null && viewedPage != null && // + if (data.file().getWorkflowStatus() != WorkflowStatus.APPROVED && lastChange != null && lastChange.getDateTime() != null && viewedPage != null && // viewedPage.isBefore(lastChange.getDateTime())) { hasUpdates = true; } @@ -169,6 +177,25 @@ public class AnalysisFlagsCalculationService { } + private FlagCalculationData getFlagCalculationData(String dossierId, String fileId) { + + var file = fileStatusPersistenceService.getStatus(fileId); + var entityLog = entityLogService.getEntityLog(dossierId, fileId, true); + var viewedPagesForCurrentAssignee = viewedPagesPersistenceService.findViewedPages(fileId, file.getAssignee()) + .stream() + .collect(Collectors.toMap(ViewedPageEntity::getPage, ViewedPageEntity::getViewedTime)); + + return new FlagCalculationData(file, entityLog, viewedPagesForCurrentAssignee); + } + + + public record FlagCalculationData( + FileEntity file, EntityLog entityLog, Map viewedPages + ) { + + } + + private void addIdsToTrace(String dossierId, String fileId) { if (observationRegistry.getCurrentObservation() != null) { diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogMergeService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogMergeService.java index b4495bd71..bae486e92 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogMergeService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogMergeService.java @@ -78,46 +78,42 @@ public class EntityLogMergeService { Map> allManualChanges = groupManualChanges(unprocessedManualRedactions); - List entityLogEntries = new LinkedList<>(entityLog.getEntityLogEntry()); + List mergedEntityLogEntries = merge(unprocessedManualRedactions, entityLog.getEntityLogEntry(), dossier, analysisNumber, allManualChanges); - merge(unprocessedManualRedactions, entityLog.getEntityLogEntry(), dossier, analysisNumber, entityLogEntries, allManualChanges); - - entityLog.setEntityLogEntry(entityLogEntries); + entityLog.setEntityLogEntry(mergedEntityLogEntries); return entityLog; } @Observed(name = "EntityLogMergeService", contextualName = "merge-entity-log-entries") - public List mergeEntityLogEntries(ManualRedactions unprocessedManualRedactions, List entityLogEntryIds, DossierEntity dossier, String fileId) { + public List mergeEntityLogEntries(ManualRedactions unprocessedManualRedactions, List entityLogEntries, DossierEntity dossier, String fileId) { final int analysisNumber = entityLogMongoService.findLatestAnalysisNumber(dossier.getId(), fileId) .orElseThrow(() -> new BadRequestException("Can't load analysis number")); Map> allManualChanges = groupManualChanges(unprocessedManualRedactions); - List entityLogEntries = entityLogMongoService.findEntityLogEntriesByIds(dossier.getId(), fileId, entityLogEntryIds); - - merge(unprocessedManualRedactions, entityLogEntries, dossier, analysisNumber, entityLogEntries, allManualChanges); - - return entityLogEntries; + return merge(unprocessedManualRedactions, entityLogEntries, dossier, analysisNumber, allManualChanges); } - private void merge(ManualRedactions unprocessedManualRedactions, - List entityLog, - DossierEntity dossier, - int analysisNumber, - List entityLogEntries, - Map> allManualChanges) { + private List merge(ManualRedactions unprocessedManualRedactions, + List entityLogEntries, + DossierEntity dossier, + int analysisNumber, + Map> allManualChanges) { Map trackLocalChangesBasedOnDictEntriesMap = new HashMap<>(); - Map addedLocalManualEntries = buildUnprocessedLocalManualRedactions(unprocessedManualRedactions, entityLog, dossier, analysisNumber)// + List mergedEntityLogEntries = new LinkedList<>(entityLogEntries); + Map addedLocalManualEntries = buildUnprocessedLocalManualRedactions(unprocessedManualRedactions, entityLogEntries, dossier, analysisNumber)// .collect(Collectors.toMap(EntityLogEntry::getId, Function.identity())); - entityLogEntries.addAll(addedLocalManualEntries.values()); - buildPendingDictionaryChanges(unprocessedManualRedactions).forEach(entityLogEntries::add); - processEntityLogEntries(dossier, entityLogEntries, addedLocalManualEntries, analysisNumber, allManualChanges, trackLocalChangesBasedOnDictEntriesMap); + mergedEntityLogEntries.addAll(addedLocalManualEntries.values()); + buildPendingDictionaryChanges(unprocessedManualRedactions).forEach(mergedEntityLogEntries::add); + + processEntityLogEntries(dossier, mergedEntityLogEntries, addedLocalManualEntries, analysisNumber, allManualChanges, trackLocalChangesBasedOnDictEntriesMap); adjustEntityLogEntriesAfterLocalChangesBasedOnDict(entityLogEntries, trackLocalChangesBasedOnDictEntriesMap, analysisNumber); + return mergedEntityLogEntries; } @@ -397,7 +393,7 @@ public class EntityLogMergeService { entityLogEntry.setState(EntryState.IGNORED); //special case, only for add local and remove only - if (!entityLogEntry.getEngines().isEmpty() && Set.of(Engine.MANUAL).containsAll(entityLogEntry.getEngines())) { + if (entityLogEntry.getEngines().equals(Set.of(Engine.MANUAL))) { entityLogEntry.setState(EntryState.REMOVED); change.setType(ChangeType.REMOVED); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogMongoWrapperService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogMongoWrapperService.java index 15dd91688..961dcec7f 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogMongoWrapperService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogMongoWrapperService.java @@ -37,7 +37,7 @@ public class EntityLogMongoWrapperService { if (includeUnprocessed) { DossierEntity dossier = dossierService.getDossierById(dossierId); ManualRedactions unprocessedManualRedactions = manualRedactionProviderService.getManualRedactions(fileId, ManualChangesQueryOptions.unprocessedOnlyForIds(ids)); - entityLogEntries = entityLogMergeService.mergeEntityLogEntries(unprocessedManualRedactions, entityLogEntries.stream().map(EntityLogEntry::getId).toList(), dossier, fileId); + entityLogEntries = entityLogMergeService.mergeEntityLogEntries(unprocessedManualRedactions, entityLogEntries, dossier, fileId); } return entityLogEntries; } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionMapper.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionMapper.java index ab1534ca2..fe7d93be2 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionMapper.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionMapper.java @@ -9,19 +9,17 @@ import java.util.Objects; import java.util.Set; import java.util.UUID; import java.util.function.Consumer; -import java.util.function.Function; import java.util.stream.Collectors; import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualRecategorizationEntity; import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException; -import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; import com.iqser.red.service.persistence.management.v1.processor.service.EntityLogMongoWrapperService; import com.iqser.red.service.persistence.service.v1.api.shared.model.RequestEntryPair; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine; -import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AddRedactionRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ForceRedactionRequest; @@ -96,15 +94,24 @@ public class ManualRedactionMapper { .toList(), includeUnprocessed); - Map iddToEntityLogEntryMap = entityLogEntries.stream() - .collect(Collectors.toMap(EntityLogEntry::getId, Function.identity())); + Map> idsToEntityLogEntryMap = entityLogEntries.stream() + .collect(Collectors.groupingBy(EntityLogEntry::getId)); List> requests = new ArrayList<>(); for (var removeRedactionRequest : removeRedactionRequests) { - EntityLogEntry entityLogEntry = iddToEntityLogEntryMap.get(removeRedactionRequest.getAnnotationId()); + if (!idsToEntityLogEntryMap.containsKey(removeRedactionRequest.getAnnotationId())) { + continue; + } + + List entityLogEntriesWithId = idsToEntityLogEntryMap.get(removeRedactionRequest.getAnnotationId()); + + for (EntityLogEntry entityLogEntry : entityLogEntriesWithId) { + + if (invalidDictionaryRequest(removeRedactionRequest, entityLogEntry) || invalidLocalRequest(removeRedactionRequest, entityLogEntry)) { + continue; + } - if (entityLogEntry != null) { var request = RemoveRedactionRequest.builder() .annotationId(removeRedactionRequest.getAnnotationId()) .user(KeycloakSecurity.getUserId()) @@ -128,6 +135,24 @@ public class ManualRedactionMapper { } + private static boolean invalidLocalRequest(RemoveRedactionRequestModel removeRedactionRequest, EntityLogEntry entityLogEntry) { + + return !isDictionaryRequest(removeRedactionRequest) && entityLogEntry.getState().equals(EntryState.REMOVED); + } + + + private static boolean isDictionaryRequest(RemoveRedactionRequestModel removeRedactionRequest) { + + return removeRedactionRequest.isRemoveFromDictionary() || removeRedactionRequest.isRemoveFromAllDossiers(); + } + + + private static boolean invalidDictionaryRequest(RemoveRedactionRequestModel removeRedactionRequest, EntityLogEntry entityLogEntry) { + + return isDictionaryRequest(removeRedactionRequest) && entityLogEntry.getState().equals(EntryState.PENDING); + } + + public List> toForceRedactionRequestList(String dossierId, String fileId, Set forceRedactionRequests, @@ -208,15 +233,24 @@ public class ManualRedactionMapper { .toList(), includeUnprocessed); - Map annotationIdToEntityLogEntryMap = entityLogEntries.stream() - .collect(Collectors.toMap(EntityLogEntry::getId, Function.identity())); + Map> annotationIdToEntityLogEntryMap = entityLogEntries.stream() + .collect(Collectors.groupingBy(EntityLogEntry::getId)); List> requests = new ArrayList<>(); for (RecategorizationRequestModel recategorizationRequest : recategorizationRequests) { - EntityLogEntry entityLogEntry = annotationIdToEntityLogEntryMap.get(recategorizationRequest.getAnnotationId()); + List entityLogEntriesById = annotationIdToEntityLogEntryMap.get(recategorizationRequest.getAnnotationId()); + + if (entityLogEntriesById == null) { + continue; + } + + for (EntityLogEntry entityLogEntry : entityLogEntriesById) { + + if (invalidDictionaryRequest(recategorizationRequest, entityLogEntry) || invalidLocalRequest(recategorizationRequest, entityLogEntry)) { + continue; + } - if (entityLogEntry != null) { String changedValue; String changedTypeId; String uuid = UUID.randomUUID().toString(); @@ -271,6 +305,24 @@ public class ManualRedactionMapper { } + private static boolean invalidLocalRequest(RecategorizationRequestModel removeRedactionRequest, EntityLogEntry entityLogEntry) { + + return !isDictionaryRequest(removeRedactionRequest) && entityLogEntry.getState().equals(EntryState.REMOVED); + } + + + private static boolean isDictionaryRequest(RecategorizationRequestModel removeRedactionRequest) { + + return removeRedactionRequest.isAddToDictionary() || removeRedactionRequest.isAddToAllDossiers(); + } + + + private static boolean invalidDictionaryRequest(RecategorizationRequestModel removeRedactionRequest, EntityLogEntry entityLogEntry) { + + return isDictionaryRequest(removeRedactionRequest) && entityLogEntry.getState().equals(EntryState.PENDING); + } + + private void checkSectionLength(String changedSection) { if (changedSection == null) { @@ -358,16 +410,6 @@ public class ManualRedactionMapper { } - private static EntityLogEntry getEntityLogEntry(EntityLog entityLog, String annotationId) { - - return entityLog.getEntityLogEntry() - .stream() - .filter(entry -> entry.getId().equals(annotationId)) - .findFirst() - .orElseThrow(() -> new NotFoundException("Annotation does not exist in entity log.")); - } - - public static DictionaryEntryType getDictionaryEntryType(EntityLogEntry entityLogEntry) { if (entityLogEntry.getEntryType().equals(EntryType.FALSE_RECOMMENDATION)) { diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java index e5cfa10d0..9ec7f1e5e 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java @@ -389,7 +389,7 @@ public interface FileRepository extends JpaRepository { select distinct f.id, f.dossierId from FileEntity f left join ViewedPageEntity v on f.id = v.file.id and v.id.userId = f.assignee - where f.deleted is NULL and f.hardDeletedTime is NULL + where f.deleted is NULL and f.hardDeletedTime is NULL and f.processingStatus = 'PROCESSED' and ((f.lastFlagCalculation is NULL and f.lastProcessed is not NULL) or f.lastManualChangeDate > f.lastFlagCalculation or f.lastProcessed > f.lastFlagCalculation or f.lastFlagCalculation < v.viewedTime) """) List getFileIdentifiersWhereAnalysisFlagCalculationIsRequired(); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/AnalysisFlagsCalculationMessageReceiver.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/AnalysisFlagsCalculationMessageReceiver.java index cd404f848..bbf9dfbde 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/AnalysisFlagsCalculationMessageReceiver.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/AnalysisFlagsCalculationMessageReceiver.java @@ -33,7 +33,7 @@ public class AnalysisFlagsCalculationMessageReceiver { public void receiveAnalysisFlagCalculationMessage(Message message) { var analysisFlagCalculationMessage = mapper.readValue(message.getBody(), AnalysisFlagCalculationMessage.class); - log.info("Calculating Flags for fileId {} and dossierId {}", analysisFlagCalculationMessage.getFileId(), analysisFlagCalculationMessage.getDossierId()); + log.debug("Calculating Flags for fileId {} and dossierId {}", analysisFlagCalculationMessage.getFileId(), analysisFlagCalculationMessage.getDossierId()); analysisFlagsCalculationService.calculateFlags(analysisFlagCalculationMessage.getDossierId(), analysisFlagCalculationMessage.getFileId()); } diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/AnalysisFlagCalculationServiceTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/AnalysisFlagCalculationServiceTest.java index 12bab19eb..1be0880b1 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/AnalysisFlagCalculationServiceTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/AnalysisFlagCalculationServiceTest.java @@ -36,9 +36,10 @@ public class AnalysisFlagCalculationServiceTest extends AbstractPersistenceServe EntityLog entityLog = objectMapper.readValue(file.getInputStream(), EntityLog.class); - AnalysisFlags analysisFlags = analysisFlagsCalculationService.calculateFlags(entityLog, - Map.of(1, OffsetDateTime.now()), - FileEntity.builder().workflowStatus(WorkflowStatus.UNDER_REVIEW).build()); + var data = new AnalysisFlagsCalculationService.FlagCalculationData(FileEntity.builder().workflowStatus(WorkflowStatus.UNDER_REVIEW).build(), + entityLog, + Map.of(1, OffsetDateTime.now())); + AnalysisFlags analysisFlags = analysisFlagsCalculationService.calculateFlags(data); assertTrue(analysisFlags.hasRedactions()); }