Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

RED-6864 - Optimization of Upload and Download for Large Files in Azure Blob... #82

Merged
corina.olariu.ext1 merged 3 commits from RED-6864-dlq into master 2023-08-18 09:43:58 +02:00
Conversation 12 Commits 3 Files Changed 2 +14 -5
2 changed files with 14 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadPreparationService.java
View File

@ -141,8 +141,12 @@ public class DownloadPreparationService {
public void processingRedactionResultMessage(RedactionResultMessage redactionResultMessage) {
String downloadId = redactionResultMessage.getDownloadId();
long totalFiles = downloadStatusPersistenceService.getStatus(downloadId).getFiles().size();
List<RedactionFileResult> redactionFileResults = redactionFileResultsMap.get(downloadId);
if (redactionFileResults == null) {
log.info("The creation of download has finished for downloadId: {} ", downloadId);
return;
}
long totalFiles = downloadStatusPersistenceService.getStatus(downloadId).getFiles().size();
Optional<RedactionFileResult> resultOptional = redactionFileResults.stream().filter(r -> r.getFileId().equals(redactionResultMessage.getFileId())).findFirst();
resultOptional.ifPresent(redactionFileResult -> {
redactionFileResult.setRedactionResultDetailList(redactionResultMessage.getRedactionResultDetails());
@ -160,18 +164,22 @@ public class DownloadPreparationService {
public void checkForRetryProcess(String downloadId, String fileId, boolean unapprovedFile) {
List<RedactionFileResult> redactionFileResults = redactionFileResultsMap.get(downloadId);
if (redactionFileResults == null) {
log.info("The creation of download has finished for downloadId: {} ", downloadId);
return;
}
Optional<RedactionFileResult> resultOptional = redactionFileResults.stream().filter(r -> r.getFileId().equals(fileId)).findFirst();
if (resultOptional.isPresent()) {
RedactionFileResult result = resultOptional.get();
if (result.getRetryCounter() >= MAX_RETRY) { // update download status
log.info("Failed download after max retries for downloadId: {}", downloadId);
log.info("Failed download after max retries: {} for downloadId: {}, set the download status to FAILED", result.getRetryCounter(), downloadId);
downloadStatusPersistenceService.updateStatus(downloadId, DownloadStatusValue.FAILED);
} else { // retry to send it again
result.increaseRetryCounter();
var downloadStatus = downloadStatusPersistenceService.getStatus(downloadId);
RedactionMessage.RedactionMessageBuilder messageBuilder = this.generateGeneralRedactionMessage(downloadId, downloadStatus);
RedactionMessage message = messageBuilder.fileId(fileId).unapprovedFile(unapprovedFile).build();
log.info("Resending redaction request for downloadId:{} fileId: {} to pdftron-redaction-queue", downloadId, fileId);
log.info("Resending redaction request for downloadId:{} fileId: {} to {}", downloadId, fileId, MessagingConfiguration.PDFTRON_QUEUE);
rabbitTemplate.convertAndSend(MessagingConfiguration.PDFTRON_QUEUE, message);
}
}
@ -190,6 +198,7 @@ public class DownloadPreparationService {
storeZipFile(downloadStatus, fileSystemBackedArchiver);
updateStatusToReady(downloadStatus, fileSystemBackedArchiver);
redactionFileResultsMap.remove(downloadId);
notificationPersistenceService.insertNotification(AddNotificationRequest.builder()
.userId(downloadStatus.getUserId())

4
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/RedactionDlqMessageReceiver.java
View File

@ -23,7 +23,6 @@ import lombok.extern.slf4j.Slf4j;
@Service
@RequiredArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true)
@RabbitListener(queues = MessagingConfiguration.PDFTRON_DLQ)
public class RedactionDlqMessageReceiver {
private static final String LINE_SEPARATOR = System.lineSeparator();
@ -34,6 +33,7 @@ public class RedactionDlqMessageReceiver {
@RabbitHandler
@RabbitListener(queues = MessagingConfiguration.PDFTRON_DLQ)
public void receive(Message message) throws IOException {
// Since we receive different message types here, we do not convert to an object here;
@ -52,7 +52,7 @@ public class RedactionDlqMessageReceiver {
try {
fileId = jsonNode.findValue("fileId").asText();
var unapproved = jsonNode.findValue("unapprovedFile");
log.info("Received a dead message with downloadId: {}, fileId: {} retry", downloadId, fileId);
log.info("Received a dead message with downloadId: {}, fileId: {} check for retry", downloadId, fileId);
downloadPreparationService.checkForRetryProcess(downloadId, fileId, unapproved.asBoolean());
} catch (Exception e) {