Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

RED-6864 - Optimization of Upload and Download for Large Files in Azure Blob Storage and AWS S3/MinIO

Browse Source 
- remove the entry from map when download finished
- check for null in case the download process has finished
This commit is contained in:
Corina Olariu 2023-08-18 10:26:23 +03:00
parent 11958a4099
commit 896f0a6e5d
2 changed files with 13 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadPreparationService.java
View File

@ -141,8 +141,12 @@ public class DownloadPreparationService {
public void processingRedactionResultMessage(RedactionResultMessage redactionResultMessage) { public void processingRedactionResultMessage(RedactionResultMessage redactionResultMessage) {
String downloadId = redactionResultMessage.getDownloadId(); String downloadId = redactionResultMessage.getDownloadId();
long totalFiles = downloadStatusPersistenceService.getStatus(downloadId).getFiles().size();
List<RedactionFileResult> redactionFileResults = redactionFileResultsMap.get(downloadId); List<RedactionFileResult> redactionFileResults = redactionFileResultsMap.get(downloadId);
if (redactionFileResults == null) {
log.info("The creation of download has finished for downloadId: {} ", downloadId);
return;
}
long totalFiles = downloadStatusPersistenceService.getStatus(downloadId).getFiles().size();
Optional<RedactionFileResult> resultOptional = redactionFileResults.stream().filter(r -> r.getFileId().equals(redactionResultMessage.getFileId())).findFirst(); Optional<RedactionFileResult> resultOptional = redactionFileResults.stream().filter(r -> r.getFileId().equals(redactionResultMessage.getFileId())).findFirst();
resultOptional.ifPresent(redactionFileResult -> { resultOptional.ifPresent(redactionFileResult -> {
redactionFileResult.setRedactionResultDetailList(redactionResultMessage.getRedactionResultDetails()); redactionFileResult.setRedactionResultDetailList(redactionResultMessage.getRedactionResultDetails());
@ -160,18 +164,22 @@ public class DownloadPreparationService {
public void checkForRetryProcess(String downloadId, String fileId, boolean unapprovedFile) { public void checkForRetryProcess(String downloadId, String fileId, boolean unapprovedFile) {
List<RedactionFileResult> redactionFileResults = redactionFileResultsMap.get(downloadId); List<RedactionFileResult> redactionFileResults = redactionFileResultsMap.get(downloadId);
if (redactionFileResults == null) {
log.info("The creation of download has finished for downloadId: {} ", downloadId);
return;
}
Optional<RedactionFileResult> resultOptional = redactionFileResults.stream().filter(r -> r.getFileId().equals(fileId)).findFirst(); Optional<RedactionFileResult> resultOptional = redactionFileResults.stream().filter(r -> r.getFileId().equals(fileId)).findFirst();
if (resultOptional.isPresent()) { if (resultOptional.isPresent()) {
RedactionFileResult result = resultOptional.get(); RedactionFileResult result = resultOptional.get();
if (result.getRetryCounter() >= MAX_RETRY) { // update download status if (result.getRetryCounter() >= MAX_RETRY) { // update download status
log.info("Failed download after max retries for downloadId: {}", downloadId); log.info("Failed download after max retries: {} for downloadId: {}, set the download status to FAILED", result.getRetryCounter(), downloadId);
downloadStatusPersistenceService.updateStatus(downloadId, DownloadStatusValue.FAILED); downloadStatusPersistenceService.updateStatus(downloadId, DownloadStatusValue.FAILED);
} else { // retry to send it again } else { // retry to send it again
result.increaseRetryCounter(); result.increaseRetryCounter();
var downloadStatus = downloadStatusPersistenceService.getStatus(downloadId); var downloadStatus = downloadStatusPersistenceService.getStatus(downloadId);
RedactionMessage.RedactionMessageBuilder messageBuilder = this.generateGeneralRedactionMessage(downloadId, downloadStatus); RedactionMessage.RedactionMessageBuilder messageBuilder = this.generateGeneralRedactionMessage(downloadId, downloadStatus);
RedactionMessage message = messageBuilder.fileId(fileId).unapprovedFile(unapprovedFile).build(); RedactionMessage message = messageBuilder.fileId(fileId).unapprovedFile(unapprovedFile).build();
log.info("Resending redaction request for downloadId:{} fileId: {} to pdftron-redaction-queue", downloadId, fileId); log.info("Resending redaction request for downloadId:{} fileId: {} to {}", downloadId, fileId, MessagingConfiguration.PDFTRON_QUEUE);
rabbitTemplate.convertAndSend(MessagingConfiguration.PDFTRON_QUEUE, message); rabbitTemplate.convertAndSend(MessagingConfiguration.PDFTRON_QUEUE, message);
} }
} }
@ -190,6 +198,7 @@ public class DownloadPreparationService {
storeZipFile(downloadStatus, fileSystemBackedArchiver); storeZipFile(downloadStatus, fileSystemBackedArchiver);
updateStatusToReady(downloadStatus, fileSystemBackedArchiver); updateStatusToReady(downloadStatus, fileSystemBackedArchiver);
redactionFileResultsMap.remove(downloadId);
notificationPersistenceService.insertNotification(AddNotificationRequest.builder() notificationPersistenceService.insertNotification(AddNotificationRequest.builder()
.userId(downloadStatus.getUserId()) .userId(downloadStatus.getUserId())

2
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/RedactionDlqMessageReceiver.java
View File

@ -52,7 +52,7 @@ public class RedactionDlqMessageReceiver {
try { try {
fileId = jsonNode.findValue("fileId").asText(); fileId = jsonNode.findValue("fileId").asText();
var unapproved = jsonNode.findValue("unapprovedFile"); var unapproved = jsonNode.findValue("unapprovedFile");
log.info("Received a dead message with downloadId: {}, fileId: {} retry", downloadId, fileId); log.info("Received a dead message with downloadId: {}, fileId: {} check for retry", downloadId, fileId);
downloadPreparationService.checkForRetryProcess(downloadId, fileId, unapproved.asBoolean()); downloadPreparationService.checkForRetryProcess(downloadId, fileId, unapproved.asBoolean());
} catch (Exception e) { } catch (Exception e) {