From 11958a4099ae3e095b61deee7e15c78f8cae44f4 Mon Sep 17 00:00:00 2001 From: Corina Olariu Date: Thu, 17 Aug 2023 16:50:09 +0300 Subject: [PATCH 1/2] RED-6864 - Optimization of Upload and Download for Large Files in Azure Blob Storage and AWS S3/MinIO - add @RabbitListener to the method --- .../processor/service/download/RedactionDlqMessageReceiver.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/RedactionDlqMessageReceiver.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/RedactionDlqMessageReceiver.java index f6a480a1c..eac320b1b 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/RedactionDlqMessageReceiver.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/RedactionDlqMessageReceiver.java @@ -23,7 +23,6 @@ import lombok.extern.slf4j.Slf4j; @Service @RequiredArgsConstructor @FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true) -@RabbitListener(queues = MessagingConfiguration.PDFTRON_DLQ) public class RedactionDlqMessageReceiver { private static final String LINE_SEPARATOR = System.lineSeparator(); @@ -34,6 +33,7 @@ public class RedactionDlqMessageReceiver { @RabbitHandler + @RabbitListener(queues = MessagingConfiguration.PDFTRON_DLQ) public void receive(Message message) throws IOException { // Since we receive different message types here, we do not convert to an object here; -- 2.47.2 From 896f0a6e5d3fabd7201a946bdb914a799d62eebb Mon Sep 17 00:00:00 2001 From: Corina Olariu Date: Fri, 18 Aug 2023 10:26:23 +0300 Subject: [PATCH 2/2] RED-6864 - Optimization of Upload and Download for Large Files in Azure Blob Storage and AWS S3/MinIO - remove the entry from map when download finished - check for null in case the download process has finished --- .../download/DownloadPreparationService.java | 15 ++++++++++++--- .../download/RedactionDlqMessageReceiver.java | 2 +- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadPreparationService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadPreparationService.java index 3b1a68563..9ebe1ab82 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadPreparationService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadPreparationService.java @@ -141,8 +141,12 @@ public class DownloadPreparationService { public void processingRedactionResultMessage(RedactionResultMessage redactionResultMessage) { String downloadId = redactionResultMessage.getDownloadId(); - long totalFiles = downloadStatusPersistenceService.getStatus(downloadId).getFiles().size(); List redactionFileResults = redactionFileResultsMap.get(downloadId); + if (redactionFileResults == null) { + log.info("The creation of download has finished for downloadId: {} ", downloadId); + return; + } + long totalFiles = downloadStatusPersistenceService.getStatus(downloadId).getFiles().size(); Optional resultOptional = redactionFileResults.stream().filter(r -> r.getFileId().equals(redactionResultMessage.getFileId())).findFirst(); resultOptional.ifPresent(redactionFileResult -> { redactionFileResult.setRedactionResultDetailList(redactionResultMessage.getRedactionResultDetails()); @@ -160,18 +164,22 @@ public class DownloadPreparationService { public void checkForRetryProcess(String downloadId, String fileId, boolean unapprovedFile) { List redactionFileResults = redactionFileResultsMap.get(downloadId); + if (redactionFileResults == null) { + log.info("The creation of download has finished for downloadId: {} ", downloadId); + return; + } Optional resultOptional = redactionFileResults.stream().filter(r -> r.getFileId().equals(fileId)).findFirst(); if (resultOptional.isPresent()) { RedactionFileResult result = resultOptional.get(); if (result.getRetryCounter() >= MAX_RETRY) { // update download status - log.info("Failed download after max retries for downloadId: {}", downloadId); + log.info("Failed download after max retries: {} for downloadId: {}, set the download status to FAILED", result.getRetryCounter(), downloadId); downloadStatusPersistenceService.updateStatus(downloadId, DownloadStatusValue.FAILED); } else { // retry to send it again result.increaseRetryCounter(); var downloadStatus = downloadStatusPersistenceService.getStatus(downloadId); RedactionMessage.RedactionMessageBuilder messageBuilder = this.generateGeneralRedactionMessage(downloadId, downloadStatus); RedactionMessage message = messageBuilder.fileId(fileId).unapprovedFile(unapprovedFile).build(); - log.info("Resending redaction request for downloadId:{} fileId: {} to pdftron-redaction-queue", downloadId, fileId); + log.info("Resending redaction request for downloadId:{} fileId: {} to {}", downloadId, fileId, MessagingConfiguration.PDFTRON_QUEUE); rabbitTemplate.convertAndSend(MessagingConfiguration.PDFTRON_QUEUE, message); } } @@ -190,6 +198,7 @@ public class DownloadPreparationService { storeZipFile(downloadStatus, fileSystemBackedArchiver); updateStatusToReady(downloadStatus, fileSystemBackedArchiver); + redactionFileResultsMap.remove(downloadId); notificationPersistenceService.insertNotification(AddNotificationRequest.builder() .userId(downloadStatus.getUserId()) diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/RedactionDlqMessageReceiver.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/RedactionDlqMessageReceiver.java index eac320b1b..2356adf4c 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/RedactionDlqMessageReceiver.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/RedactionDlqMessageReceiver.java @@ -52,7 +52,7 @@ public class RedactionDlqMessageReceiver { try { fileId = jsonNode.findValue("fileId").asText(); var unapproved = jsonNode.findValue("unapprovedFile"); - log.info("Received a dead message with downloadId: {}, fileId: {} retry", downloadId, fileId); + log.info("Received a dead message with downloadId: {}, fileId: {} check for retry", downloadId, fileId); downloadPreparationService.checkForRetryProcess(downloadId, fileId, unapproved.asBoolean()); } catch (Exception e) { -- 2.47.2