From 1f89d64d811e5de9a101e3e7a02c5057775f7ee8 Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Tue, 15 Aug 2023 13:17:32 +0200 Subject: [PATCH 1/3] DM-357: always schedule reanalysis, when any manual redaction happens * refactored code slightly --- .../service/ManualRedactionService.java | 67 ++++++------------- .../model/annotations/ManualAddResponse.java | 1 + 2 files changed, 20 insertions(+), 48 deletions(-) diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ManualRedactionService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ManualRedactionService.java index c1e2b6af5..a3525d3c7 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ManualRedactionService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ManualRedactionService.java @@ -70,7 +70,6 @@ import lombok.extern.slf4j.Slf4j; public class ManualRedactionService { private final DossierPersistenceService dossierPersistenceService; - private final DossierTemplatePersistenceService dossierTemplatePersistenceService; private final AddRedactionPersistenceService addRedactionPersistenceService; private final RemoveRedactionPersistenceService removeRedactionPersistenceService; private final ForceRedactionPersistenceService forceRedactionPersistenceService; @@ -85,8 +84,6 @@ public class ManualRedactionService { private final ManualRedactionProviderService manualRedactionProviderService; private final AnalysisFlagsCalculationService analysisFlagsCalculationService; private final StopwordService stopwordService; - private final RabbitTemplate rabbitTemplate; - private final ObjectMapper objectMapper; private final RedactionLogService redactionLogService; private final DictionaryManagementService dictionaryManagementService; private final HashFunction hashFunction = Hashing.murmur3_128(); @@ -98,12 +95,13 @@ public class ManualRedactionService { var response = new ArrayList(); dossierPersistenceService.getAndValidateDossier(dossierId); - var actionPerformed = false; + // validate add to dossier template dictionaries addRedactionRequests.forEach(request -> dictionaryManagementService.validateAddRemoveToDossierTemplateDictionary(request.getDossierTemplateTypeId(), request.isAddToDictionary(), request.isAddToAllDossiers())); + boolean actionPerformed = false; for (var addRedactionRequest : addRedactionRequests) { if (addRedactionRequest.isAddToDictionary()) { validateDictionary(addRedactionRequest); @@ -118,7 +116,7 @@ public class ManualRedactionService { commentId = addComment(fileId, annotationId, addRedactionRequest.getComment(), addRedactionRequest.getUser()).getId(); } - var addedToDictionary = handleAddToDictionary(fileId, + handleAddToDictionary(fileId, annotationId, addRedactionRequest.getDossierTemplateTypeId(), addRedactionRequest.getValue(), @@ -129,28 +127,14 @@ public class ManualRedactionService { dossierId, addRedactionRequest.getDictionaryEntryType()); - actionPerformed = actionPerformed || addedToDictionary; - response.add(ManualAddResponse.builder().annotationId(annotationId).commentId(commentId).build()); + actionPerformed = actionPerformed || addRedactionRequest.getStatus().equals(AnnotationStatus.APPROVED); } analysisFlagsCalculationService.calculateFlags(dossierId, fileId); if (actionPerformed) { - // in case of add to dict, there is no need to process surrounding text reprocess(dossierId, fileId); - } else { - - var manualTextRedactions = response.stream() - .map(r -> manualRedactionProviderService.getAddRedaction(fileId, r.getAnnotationId())) - .filter(m -> !m.isAddToDictionary() && !m.isRectangle()) - .toList(); - - if (!manualTextRedactions.isEmpty()) { - ManualRedactions manualRedactions = ManualRedactions.builder().entriesToAdd(new HashSet<>(manualTextRedactions)).build(); - addManualRedactionToAnalysisQueue(dossierId, fileId, manualRedactions); - } - } return response; @@ -244,20 +228,6 @@ public class ManualRedactionService { } - private void addManualRedactionToAnalysisQueue(String dossierId, String fileId, ManualRedactions manualRedactions) { - - fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.SURROUNDING_TEXT_PROCESSING); - - var analyseRequest = AnalyzeRequest.builder().messageType(MessageType.SURROUNDING_TEXT).dossierId(dossierId).fileId(fileId).manualRedactions(manualRedactions).build(); - - rabbitTemplate.convertAndSend(MessagingConfiguration.REDACTION_QUEUE, analyseRequest, message -> { - message.getMessageProperties().setPriority(1); - return message; - }); - - } - - private void removeFromDictionary(String typeId, String value, String dossierId, String fileId, DictionaryEntryType dictionaryEntryType) { try { @@ -286,7 +256,7 @@ public class ManualRedactionService { var dossier = dossierPersistenceService.getAndValidateDossier(dossierId); RedactionLog redactionLog = fileManagementStorageService.getRedactionLog(dossier.getId(), fileId); - var actionPerformed = false; + var requiresReAnalysis = false; var manualRedactions = manualRedactionProviderService.getManualRedactions(fileId); //validate removing from dossier template dictionary @@ -314,18 +284,16 @@ public class ManualRedactionService { if (removeRedactionRequest.getComment() != null) { commentId = addComment(fileId, removeRedactionRequest.getAnnotationId(), removeRedactionRequest.getComment(), removeRedactionRequest.getUser()).getId(); } - if (!removeRedactionRequest.isRemoveFromDictionary() && AnnotationStatus.APPROVED.equals(removeRedactionRequest.getStatus())) { - RedactionLogEntry redactionLogEntry = null; - boolean requiresAnalysis = false; + boolean matchingEntryFound; try { - redactionLogEntry = getRedactionLogEntry(redactionLog, removeRedactionRequest.getAnnotationId()); - requiresAnalysis = redactionLogEntry.isHint(); + getRedactionLogEntry(redactionLog, removeRedactionRequest.getAnnotationId()); + matchingEntryFound = true; } catch (NotFoundException e) { - requiresAnalysis = false; + matchingEntryFound = false; } - actionPerformed = actionPerformed || requiresAnalysis; - if (!requiresAnalysis && idRemoval.isApproved()) { + requiresReAnalysis = requiresReAnalysis || matchingEntryFound; + if (!matchingEntryFound && idRemoval.isApproved()) { removeRedactionPersistenceService.markAsProcessed(idRemoval); } } @@ -343,13 +311,13 @@ public class ManualRedactionService { removeRedactionPersistenceService.markAsProcessed(idRemoval); } - actionPerformed = actionPerformed || removedFromDictionary; + requiresReAnalysis = requiresReAnalysis || removedFromDictionary; response.add(ManualAddResponse.builder().annotationId(removeRedactionRequest.getAnnotationId()).commentId(commentId).build()); } } - if (actionPerformed) { + if (requiresReAnalysis) { reprocess(dossierId, fileId); } @@ -458,7 +426,7 @@ public class ManualRedactionService { commentId = addComment(fileId, forceRedactionRequest.getAnnotationId(), forceRedactionRequest.getComment(), forceRedactionRequest.getUser()).getId(); } - actionPerformed = actionPerformed || AnnotationStatus.APPROVED.equals(forceRedactionRequest.getStatus()); + actionPerformed = actionPerformed || forceRedactionRequest.getStatus().equals(AnnotationStatus.APPROVED); response.add(ManualAddResponse.builder().annotationId(forceRedactionRequest.getAnnotationId()).commentId(commentId).build()); } @@ -514,7 +482,7 @@ public class ManualRedactionService { imageRecategorizationRequest.getUser()).getId(); } - actionPerformed = actionPerformed || !AnnotationStatus.REQUESTED.equals(imageRecategorizationRequest.getStatus()); + actionPerformed = actionPerformed || imageRecategorizationRequest.getStatus().equals(AnnotationStatus.APPROVED); response.add(ManualAddResponse.builder().annotationId(imageRecategorizationRequest.getAnnotationId()).commentId(commentId).build()); } @@ -745,7 +713,10 @@ public class ManualRedactionService { addToDictionary(typeId, newValue, dossierId, fileId, dictionaryEntryType); typeIdsOfModifiedDictionaries.add(typeId); - resizeRedactionPersistenceService.updateStatus(resizeRedaction.getId().getFileId(), resizeRedaction.getId().getAnnotationId(), resizeRedaction.getStatus(), typeIdsOfModifiedDictionaries); + resizeRedactionPersistenceService.updateStatus(resizeRedaction.getId().getFileId(), + resizeRedaction.getId().getAnnotationId(), + resizeRedaction.getStatus(), + typeIdsOfModifiedDictionaries); } } diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/ManualAddResponse.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/ManualAddResponse.java index 5a7dcdb19..24be9d9b6 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/ManualAddResponse.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/ManualAddResponse.java @@ -10,6 +10,7 @@ import lombok.NoArgsConstructor; @AllArgsConstructor @NoArgsConstructor public class ManualAddResponse { + private String annotationId; private Long commentId; -- 2.47.2 From 6956b604ccd5640903ed06e00fd7eb0b211ad448 Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Wed, 16 Aug 2023 10:28:47 +0200 Subject: [PATCH 2/3] DM-357: schedule reanalysis for every manualRedaction * silence that stupid automaticAnalysisJob spamming --- .../management/v1/processor/service/FileStatusService.java | 2 +- .../v1/processor/service/job/AutomaticAnalysisJob.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java index d4c97a921..012b76a4b 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java @@ -83,7 +83,7 @@ public class FileStatusService { var fileEntities = fileStatusPersistenceService.getAllRelevantStatusesForReanalysisScheduler(fileManagementServiceSettings.getMaxErrorRetries()); var convertedList = MagicConverter.convert(fileEntities, FileModel.class, new FileModelMapper()); - log.info("Initial list of files for scheduler contains: {} files.", fileEntities.size()); + log.debug("Initial list of files for scheduler contains: {} files.", fileEntities.size()); return reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(convertedList).stream().filter(FileModel::isAnalysisRequired).collect(Collectors.toList()); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/AutomaticAnalysisJob.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/AutomaticAnalysisJob.java index 2641a9b06..32a7e3313 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/AutomaticAnalysisJob.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/AutomaticAnalysisJob.java @@ -53,7 +53,7 @@ public class AutomaticAnalysisJob implements Job { var redactionQueueInfo = amqpAdmin.getQueueInfo(MessagingConfiguration.REDACTION_QUEUE); if (redactionQueueInfo != null) { - log.info("[Tenant:{}] Checking queue status to see if background analysis can happen. Currently {} holds {} elements and has {} consumers", + log.debug("[Tenant:{}] Checking queue status to see if background analysis can happen. Currently {} holds {} elements and has {} consumers", tenant.getTenantId(), MessagingConfiguration.REDACTION_QUEUE, redactionQueueInfo.getMessageCount(), @@ -67,7 +67,7 @@ public class AutomaticAnalysisJob implements Job { allStatuses.sort(Comparator.comparing(FileModel::getLastUpdated)); var allStatusesIterator = allStatuses.iterator(); - log.info("[Tenant:{}] Files that require reanalysis: {}", TenantContext.getTenantId(), allStatuses.size()); + log.debug("[Tenant:{}] Files that require reanalysis: {}", TenantContext.getTenantId(), allStatuses.size()); var queuedFiles = 0; -- 2.47.2 From 5e98160a58a63123e91b444b83fba34068507cf9 Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Wed, 16 Aug 2023 11:37:19 +0200 Subject: [PATCH 3/3] DM-357: remove outdated status and messageTypes * remove surrounding-text functionality --- .../service/FileStatusProcessingUpdateService.java | 7 +------ .../v1/processor/service/ManualRedactionService.java | 9 --------- .../annotations/AddRedactionPersistenceService.java | 6 ------ .../annotationentity/ManualRedactionRepository.java | 5 ----- .../service/v1/api/shared/model/MessageType.java | 5 +---- .../dossiertemplate/dossier/file/ProcessingStatus.java | 2 -- 6 files changed, 2 insertions(+), 32 deletions(-) diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusProcessingUpdateService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusProcessingUpdateService.java index 16819aa9d..1f8d07ea5 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusProcessingUpdateService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusProcessingUpdateService.java @@ -39,12 +39,7 @@ public class FileStatusProcessingUpdateService { var dossier = dossierPersistenceService.getAndValidateDossier(dossierId); switch (analyzeResult.getMessageType()) { - - case SURROUNDING_TEXT: - fileStatusService.setStatusProcessed(analyzeResult.getFileId()); - manualRedactionService.updateSurroundingText(fileId, analyzeResult.getManualRedactions()); - break; - + case ANALYSE: case REANALYSE: default: diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ManualRedactionService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ManualRedactionService.java index a3525d3c7..8679199b7 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ManualRedactionService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ManualRedactionService.java @@ -750,15 +750,6 @@ public class ManualRedactionService { } - @Transactional - public void updateSurroundingText(String fileId, ManualRedactions manualRedactions) { - - // These are marked as processed once surrounding text is computed ( TBD if this is correct ? ) - manualRedactions.getEntriesToAdd() - .forEach(e -> addRedactionPersistenceService.updateSurroundingText(new AnnotationEntityId(e.getAnnotationId(), fileId), e.getTextBefore(), e.getTextAfter())); - } - - @Transactional public void updateProcessedDate(String fileId, ManualRedactions manualRedactions) { diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/AddRedactionPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/AddRedactionPersistenceService.java index 20ae538a1..20b98dd31 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/AddRedactionPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/AddRedactionPersistenceService.java @@ -62,12 +62,6 @@ public class AddRedactionPersistenceService { } - @Transactional - public void updateSurroundingText(AnnotationEntityId id, String textBefore, String textAfter) { - - manualRedactionRepository.updateSurroundingText(id, textBefore, textAfter); - } - public ManualRedactionEntryEntity findAddRedaction(String fileId, String annotationId) { diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/annotationentity/ManualRedactionRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/annotationentity/ManualRedactionRepository.java index f9ca973a0..7712f10d2 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/annotationentity/ManualRedactionRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/annotationentity/ManualRedactionRepository.java @@ -38,11 +38,6 @@ public interface ManualRedactionRepository extends JpaRepository findByFileIdIncludeDeletions(String fileId, boolean includeDeletions); - @Modifying - @Query("update ManualRedactionEntryEntity m set m.textBefore = :textBefore, m.textAfter = :textAfter where m.id = :id") - void updateSurroundingText(AnnotationEntityId id, String textBefore, String textAfter); - - @Modifying @Query("update ManualRedactionEntryEntity m set m.status = :newStatus, m.processedDate = :processedDate where m.id.fileId in :fileIds and m.value = :filterValue and m.addToDictionary = true and m.status = :filterStatus ") void updateStatus(Set fileIds, String filterValue, AnnotationStatus filterStatus, AnnotationStatus newStatus, OffsetDateTime processedDate); diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/MessageType.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/MessageType.java index 8dcd54303..09dc18164 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/MessageType.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/MessageType.java @@ -3,8 +3,5 @@ package com.iqser.red.service.persistence.service.v1.api.shared.model; public enum MessageType { ANALYSE, - REANALYSE, - STRUCTURE_ANALYSE, - SURROUNDING_TEXT - + REANALYSE } diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/ProcessingStatus.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/ProcessingStatus.java index 8c9f2dc06..f65ceabe1 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/ProcessingStatus.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/ProcessingStatus.java @@ -12,7 +12,6 @@ public enum ProcessingStatus { PROCESSED, PROCESSING, REPROCESS, - SURROUNDING_TEXT_PROCESSING, UNPROCESSED, FULL_PROCESSING, PRE_PROCESSING_QUEUED, @@ -20,5 +19,4 @@ public enum ProcessingStatus { PRE_PROCESSED, FIGURE_DETECTION_ANALYZING, TABLE_PARSING_ANALYZING - } -- 2.47.2