From d0571a436c6dc614a3dc07cc350c96b66bf23950 Mon Sep 17 00:00:00 2001 From: Timo Bejan Date: Mon, 8 Jan 2024 10:02:53 +0200 Subject: [PATCH] Fixed transitive dependencies and concurrent download issue RED-8175 --- .../build.gradle.kts | 8 ++-- .../configuration/MessagingConfiguration.java | 17 +++++++++ .../download/DownloadCompressingService.java | 37 +++++++++++++++++++ .../DownloadCompressionMessageReceiver.java | 34 +++++++++++++++++ .../download/DownloadDLQMessageReceiver.java | 31 +++++++++------- .../download/DownloadPreparationService.java | 3 +- .../service/job/DownloadReadyJob.java | 6 +-- .../repository/DownloadStatusRepository.java | 8 +++- .../tests/DownloadPreparationTest.java | 29 +++++++++++---- .../model/download/DownloadStatusValue.java | 1 + 10 files changed, 143 insertions(+), 31 deletions(-) create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadCompressingService.java create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadCompressionMessageReceiver.java diff --git a/persistence-service-v1/persistence-service-processor-v1/build.gradle.kts b/persistence-service-v1/persistence-service-processor-v1/build.gradle.kts index f610d0f3d..71a35cfe9 100644 --- a/persistence-service-v1/persistence-service-processor-v1/build.gradle.kts +++ b/persistence-service-v1/persistence-service-processor-v1/build.gradle.kts @@ -6,9 +6,9 @@ plugins { val springBootStarterVersion = "3.1.5" dependencies { + api(project(":persistence-service-shared-api-v1")) api(project(":persistence-service-external-api-v1")) api(project(":persistence-service-internal-api-v1")) - api(project(":persistence-service-shared-api-v1")) api("com.iqser.red.service:pdftron-redaction-service-api-v1:${rootProject.extra.get("pdftronRedactionServiceVersion")}") { exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1") exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1") @@ -47,13 +47,13 @@ dependencies { api("org.springframework.boot:spring-boot-starter-web:${springBootStarterVersion}") api("com.iqser.red.commons:spring-commons:2.1.0") api("com.iqser.red.commons:jackson-commons:2.1.0") - api("org.apache.commons:commons-compress:1.21") api("com.iqser.red.commons:storage-commons:2.45.0") + api("com.iqser.red.commons:spring-boot-starter-web-custom-commons:2.1.0") + api("com.iqser.red.commons:metric-commons:2.1.0") + api("org.apache.commons:commons-compress:1.21") api("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.14.2") api("org.postgresql:postgresql:42.2.23") api("org.apache.commons:commons-lang3:3.12.0") - api("com.iqser.red.commons:spring-boot-starter-web-custom-commons:2.1.0") - api("com.iqser.red.commons:metric-commons:2.1.0") api("com.opencsv:opencsv:5.4") api("org.springframework.cloud:spring-cloud-starter-openfeign:4.0.4") api("commons-validator:commons-validator:1.7") diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/configuration/MessagingConfiguration.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/configuration/MessagingConfiguration.java index 6e97d201b..2ddf009f2 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/configuration/MessagingConfiguration.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/configuration/MessagingConfiguration.java @@ -23,6 +23,10 @@ public class MessagingConfiguration { public static final String DOWNLOAD_QUEUE = "downloadQueue"; public static final String DOWNLOAD_DLQ = "downloadDLQ"; + public static final String DOWNLOAD_COMPRESSION_QUEUE = "download_compression_queue"; + public static final String DOWNLOAD_COMPRESSION_DLQ = "download_compression_dlq"; + + public static final String EXPORT_DOWNLOAD_QUEUE = "exportDownloadQueue"; public static final String EXPORT_DOWNLOAD_DLQ = "exportDownloadDLQ"; @@ -227,6 +231,19 @@ public class MessagingConfiguration { return QueueBuilder.durable(DOWNLOAD_DLQ).build(); } + @Bean + public Queue downloadCompressionQueue() { + + return QueueBuilder.durable(DOWNLOAD_COMPRESSION_QUEUE).withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", DOWNLOAD_COMPRESSION_DLQ).build(); + } + + + @Bean + public Queue downloadCompressionQueueDLQ() { + + return QueueBuilder.durable(DOWNLOAD_COMPRESSION_DLQ).build(); + } + @Bean public Queue exportDownloadQueue() { diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadCompressingService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadCompressingService.java new file mode 100644 index 000000000..253315d88 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadCompressingService.java @@ -0,0 +1,37 @@ +package com.iqser.red.service.persistence.management.v1.processor.service.download; + +import org.springframework.amqp.rabbit.core.RabbitTemplate; +import org.springframework.stereotype.Service; + +import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration; +import com.iqser.red.service.persistence.management.v1.processor.model.DownloadJob; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DownloadStatusRepository; +import com.iqser.red.service.persistence.service.v1.api.shared.model.download.DownloadStatusValue; +import lombok.RequiredArgsConstructor; + +@Service +@RequiredArgsConstructor +public class DownloadCompressingService { + + private final DownloadPreparationService downloadPreparationService; + private final DownloadStatusRepository downloadStatusRepository; + private final RabbitTemplate rabbitTemplate; + + + public void markDownloadForCompression(String downloadId, String userId) { + + var updated = downloadStatusRepository.updateStatusOnlyIfNotAlreadySet(downloadId, DownloadStatusValue.COMPRESSING); + + if (updated == 1) { + rabbitTemplate.convertAndSend(MessagingConfiguration.DOWNLOAD_COMPRESSION_QUEUE, DownloadJob.builder().storageId(downloadId).userId(userId).build()); + } + } + + + public void compressDownload(String downloadId) { + + downloadPreparationService.createDownload(downloadId); + downloadPreparationService.clearRedactionStatusEntries(downloadId); + } + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadCompressionMessageReceiver.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadCompressionMessageReceiver.java new file mode 100644 index 000000000..10c19e8e4 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadCompressionMessageReceiver.java @@ -0,0 +1,34 @@ +package com.iqser.red.service.persistence.management.v1.processor.service.download; + +import org.springframework.amqp.core.Message; +import org.springframework.amqp.rabbit.annotation.RabbitHandler; +import org.springframework.amqp.rabbit.annotation.RabbitListener; +import org.springframework.stereotype.Service; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration; +import com.iqser.red.service.persistence.management.v1.processor.model.DownloadJob; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +@RequiredArgsConstructor +public class DownloadCompressionMessageReceiver { + + private final DownloadCompressingService downloadCompressingService; + private final ObjectMapper objectMapper; + + + @SneakyThrows + @RabbitHandler + @RabbitListener(queues = MessagingConfiguration.DOWNLOAD_COMPRESSION_QUEUE) + public void receive(DownloadJob downloadJob) throws JsonProcessingException { + + downloadCompressingService.compressDownload(downloadJob.getStorageId()); + + } + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadDLQMessageReceiver.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadDLQMessageReceiver.java index df327d458..dc4dc003d 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadDLQMessageReceiver.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadDLQMessageReceiver.java @@ -1,13 +1,9 @@ package com.iqser.red.service.persistence.management.v1.processor.service.download; -import java.io.IOException; - -import org.springframework.amqp.core.Message; import org.springframework.amqp.rabbit.annotation.RabbitListener; import org.springframework.retry.support.RetryTemplate; import org.springframework.stereotype.Service; -import com.fasterxml.jackson.databind.ObjectMapper; import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration; import com.iqser.red.service.persistence.management.v1.processor.model.DownloadJob; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DownloadStatusPersistenceService; @@ -24,25 +20,22 @@ import lombok.extern.slf4j.Slf4j; public class DownloadDLQMessageReceiver { private final DownloadStatusPersistenceService downloadStatusPersistenceService; - private final ObjectMapper objectMapper; private final RetryTemplate retryTemplate; - @RabbitListener(queues = MessagingConfiguration.DOWNLOAD_DLQ) - public void handleDlqMessage(DownloadJob downloadJob) throws IOException { + @RabbitListener(queues = MessagingConfiguration.DOWNLOAD_COMPRESSION_DLQ) + public void handleCompressionDlqMessage(DownloadJob downloadJob) { - log.warn("Handling download job in DLQ userId: {} storageId: {} - setting status to error!", downloadJob.getUserId(), downloadJob.getStorageId()); + log.warn("Handling download compression in DLQ userId: {} storageId: {} - setting status to error!", downloadJob.getUserId(), downloadJob.getStorageId()); setDownloadFailed(downloadJob.getUserId(), downloadJob.getStorageId()); } - public void setDownloadFailed(String userId, String downloadId) { + @RabbitListener(queues = MessagingConfiguration.DOWNLOAD_DLQ) + public void handleDlqMessage(DownloadJob downloadJob) { - retryTemplate.execute(retryContext -> { - log.warn("Retrying {} time to set FAILED status for downloadJob userId: {} storageId: {}", retryContext.getRetryCount(), userId, downloadId); - downloadStatusPersistenceService.updateStatus(downloadId, DownloadStatusValue.FAILED); - return null; - }); + log.warn("Handling download job in DLQ userId: {} storageId: {} - setting status to error!", downloadJob.getUserId(), downloadJob.getStorageId()); + setDownloadFailed(downloadJob.getUserId(), downloadJob.getStorageId()); } @@ -61,4 +54,14 @@ public class DownloadDLQMessageReceiver { setDownloadFailed(reportResultMessage.getUserId(), reportResultMessage.getDownloadId()); } + + private void setDownloadFailed(String userId, String downloadId) { + + retryTemplate.execute(retryContext -> { + log.warn("Retrying {} time to set FAILED status for downloadJob userId: {} storageId: {}", retryContext.getRetryCount(), userId, downloadId); + downloadStatusPersistenceService.updateStatus(downloadId, DownloadStatusValue.FAILED); + return null; + }); + } + } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadPreparationService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadPreparationService.java index 65e583478..f259ae7f3 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadPreparationService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadPreparationService.java @@ -176,8 +176,9 @@ public class DownloadPreparationService { } - public void createDownload(List redactionFileResults, String downloadId) { + public void createDownload(String downloadId) { + var redactionFileResults = downloadRedactionFileStatusRepository.findAllByDownloadStorageId(downloadId); DownloadStatusEntity downloadStatus = downloadStatusPersistenceService.getStatus(downloadId); var storedFileInformations = getStoredFileInformation(downloadId); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/DownloadReadyJob.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/DownloadReadyJob.java index 71927f25a..ea8d44cfa 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/DownloadReadyJob.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/DownloadReadyJob.java @@ -1,5 +1,6 @@ package com.iqser.red.service.persistence.management.v1.processor.service.job; +import com.iqser.red.service.persistence.management.v1.processor.service.download.DownloadCompressingService; import com.iqser.red.service.persistence.management.v1.processor.service.download.DownloadPreparationService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DownloadStatusPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DownloadRedactionFileStatusRepository; @@ -20,7 +21,7 @@ public class DownloadReadyJob implements Job { private final DownloadStatusPersistenceService downloadStatusPersistenceService; private final DownloadRedactionFileStatusRepository downloadRedactionFileStatusRepository; - private final DownloadPreparationService downloadPreparationService; + private final DownloadCompressingService downloadCompressingService; private final TenantProvider tenantProvider; @@ -42,8 +43,7 @@ public class DownloadReadyJob implements Job { int numberOfFiles = download.getFiles().size(); var downloadRedactionFileStatus = downloadRedactionFileStatusRepository.findAllByDownloadStorageId(download.getStorageId()); if (downloadRedactionFileStatus.size() == numberOfFiles) { - downloadPreparationService.createDownload(downloadRedactionFileStatus, download.getStorageId()); - downloadPreparationService.clearRedactionStatusEntries(download.getStorageId()); + downloadCompressingService.markDownloadForCompression(download.getStorageId(),download.getUserId()); } }); }); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/DownloadStatusRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/DownloadStatusRepository.java index 4a142b236..277169e4e 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/DownloadStatusRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/DownloadStatusRepository.java @@ -7,6 +7,7 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.springframework.transaction.annotation.Transactional; import com.iqser.red.service.persistence.management.v1.processor.entity.download.DownloadStatusEntity; import com.iqser.red.service.persistence.service.v1.api.shared.model.download.DownloadStatusValue; @@ -17,7 +18,12 @@ public interface DownloadStatusRepository extends JpaRepository finalDownloadStatuses = downloadClient.getDownloadStatus().getDownloadStatus(); - assertThat(finalDownloadStatuses).hasSize(1); - DownloadStatus finalDownloadStatus = finalDownloadStatuses.get(0); - assertThat(finalDownloadStatus.getStatus()).isEqualTo(DownloadStatusValue.READY); - assertThat(finalDownloadStatus.getFileSize()).isGreaterThan(0); + downloadCompressionMessageReceiver.receive(DownloadJob.builder().storageId(firstStatus.getStorageId()).build()); + firstStatus = getFirstStatus(); + assertThat(firstStatus.getStatus()).isEqualTo(DownloadStatusValue.READY); + assertThat(firstStatus.getFileSize()).isGreaterThan(0); clearTenantContext(); } @@ -206,6 +211,14 @@ public class DownloadPreparationTest extends AbstractPersistenceServerServiceTes } + private DownloadStatus getFirstStatus() { + + List finalDownloadStatuses = downloadClient.getDownloadStatus().getDownloadStatus(); + assertThat(finalDownloadStatuses).hasSize(1); + return finalDownloadStatuses.iterator().next(); + } + + @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) private final class DossierWithSingleFile { diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/download/DownloadStatusValue.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/download/DownloadStatusValue.java index 0de2dfbc9..681dceac5 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/download/DownloadStatusValue.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/download/DownloadStatusValue.java @@ -3,6 +3,7 @@ package com.iqser.red.service.persistence.service.v1.api.shared.model.download; public enum DownloadStatusValue { QUEUED, GENERATING, + COMPRESSING, READY, FAILED }