Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

Merge branch 'RED-6805' into 'master'

Browse Source 
RED-6805: As Operation I want to see why downloads are in an ERROR state

Closes RED-6805

See merge request redactmanager/persistence-service!306
This commit is contained in:
Yannik Hampe 2024-01-18 17:10:14 +01:00
commit f9ab3a2607
9 changed files with 102 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/DownloadResource.java vendored
View File

@ -1,12 +1,7 @@
package com.iqser.red.service.persistence.service.v1.api.external.resource;
import java.util.concurrent.CompletableFuture;
import org.springframework.core.io.FileSystemResource;
import org.springframework.core.io.InputStreamResource;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
@ -72,7 +67,6 @@ public interface DownloadResource {
void downloadFile(@RequestParam(STORAGE_ID) String storageId);
@ResponseBody
@ResponseStatus(value = HttpStatus.OK)
@Operation(summary = "Returns a oneTimeToken for a requested download", description = "Use the optional \"inline\" request parameter " + "to select, if this report will be opened in the browser.")

23
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/download/DownloadStatusEntity.java
View File

@ -6,17 +6,6 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
import jakarta.persistence.Column;
import jakarta.persistence.Convert;
import jakarta.persistence.Entity;
import jakarta.persistence.EnumType;
import jakarta.persistence.Enumerated;
import jakarta.persistence.FetchType;
import jakarta.persistence.Id;
import jakarta.persistence.ManyToMany;
import jakarta.persistence.ManyToOne;
import jakarta.persistence.Table;
import org.hibernate.annotations.Fetch;
import org.hibernate.annotations.FetchMode;
@ -27,6 +16,16 @@ import com.iqser.red.service.persistence.management.v1.processor.utils.JSONDownl
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DownloadFileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.download.DownloadStatusValue;
import jakarta.persistence.Column;
import jakarta.persistence.Convert;
import jakarta.persistence.Entity;
import jakarta.persistence.EnumType;
import jakarta.persistence.Enumerated;
import jakarta.persistence.FetchType;
import jakarta.persistence.Id;
import jakarta.persistence.ManyToMany;
import jakarta.persistence.ManyToOne;
import jakarta.persistence.Table;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
import lombok.Builder;
@ -60,6 +59,8 @@ public class DownloadStatusEntity {
OffsetDateTime lastDownload;
@Column
long fileSize;
@Column
String errorCause;
@ManyToOne(fetch = FetchType.EAGER)
DossierEntity dossier;

45
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadDLQMessageReceiver.java
View File

@ -1,9 +1,13 @@
package com.iqser.red.service.persistence.management.v1.processor.service.download;
import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.X_ERROR_INFO_HEADER;
import org.springframework.amqp.core.Message;
import org.springframework.amqp.rabbit.annotation.RabbitListener;
import org.springframework.retry.support.RetryTemplate;
import org.springframework.stereotype.Service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration;
import com.iqser.red.service.persistence.management.v1.processor.model.DownloadJob;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DownloadStatusPersistenceService;
@ -12,6 +16,7 @@ import com.iqser.red.service.redaction.report.v1.api.model.ReportRequestMessage;
import com.iqser.red.service.redaction.report.v1.api.model.ReportResultMessage;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@ -21,6 +26,8 @@ public class DownloadDLQMessageReceiver {
private final DownloadStatusPersistenceService downloadStatusPersistenceService;
private final RetryTemplate retryTemplate;
private final ObjectMapper objectMapper;
private final static int MAX_ERROR_CAUSE_LENGTH = 1024;
@RabbitListener(queues = MessagingConfiguration.DOWNLOAD_COMPRESSION_DLQ)
@ -31,19 +38,35 @@ public class DownloadDLQMessageReceiver {
}
@SneakyThrows
@RabbitListener(queues = MessagingConfiguration.DOWNLOAD_DLQ)
public void handleDlqMessage(DownloadJob downloadJob) {
public void handleDlqMessage(Message failedMessage) {
log.warn("Handling download job in DLQ userId: {} storageId: {} - setting status to error!", downloadJob.getUserId(), downloadJob.getStorageId());
setDownloadFailed(downloadJob.getUserId(), downloadJob.getStorageId());
var downloadJob = objectMapper.readValue(failedMessage.getBody(), DownloadJob.class);
String errorCause = failedMessage.getMessageProperties().getHeader(X_ERROR_INFO_HEADER);
errorCause = errorCause.substring(0,MAX_ERROR_CAUSE_LENGTH-1);
if (errorCause == null) {
errorCause = "Unknown error occured in download queue!";
}
log.warn("Handling download job in DLQ userId: {} storageId: {} - setting status to error! cause: {}", downloadJob.getUserId(), downloadJob.getStorageId(), errorCause);
setDownloadFailed(downloadJob.getUserId(), downloadJob.getStorageId(), errorCause);
}
@SneakyThrows
@RabbitListener(queues = MessagingConfiguration.REPORT_DLQ)
public void handleReportDlqMessage(ReportRequestMessage reportRequestMessage) {
public void handleReportDlqMessage(Message failedMessage) {
var reportRequestMessage = objectMapper.readValue(failedMessage.getBody(), ReportRequestMessage.class);
String errorCause = failedMessage.getMessageProperties().getHeader(X_ERROR_INFO_HEADER);
if (errorCause == null) {
errorCause = "Unknown error occured in report redaction queue!";
}
else if(errorCause.length() > MAX_ERROR_CAUSE_LENGTH) {
errorCause = errorCause.substring(0,MAX_ERROR_CAUSE_LENGTH-1);
}
log.warn("Handling report request in DLQ userId: {} storageId: {} - setting status to error!", reportRequestMessage.getUserId(), reportRequestMessage.getDownloadId());
setDownloadFailed(reportRequestMessage.getUserId(), reportRequestMessage.getDownloadId());
setDownloadFailed(reportRequestMessage.getUserId(), reportRequestMessage.getDownloadId(), errorCause);
}
@ -59,7 +82,17 @@ public class DownloadDLQMessageReceiver {
retryTemplate.execute(retryContext -> {
log.warn("Retrying {} time to set FAILED status for downloadJob userId: {} storageId: {}", retryContext.getRetryCount(), userId, downloadId);
downloadStatusPersistenceService.updateStatus(downloadId, DownloadStatusValue.FAILED);
downloadStatusPersistenceService.updateStatus(downloadId, DownloadStatusValue.FAILED, "error occured in unknown queue");
return null;
});
}
private void setDownloadFailed(String userId, String downloadId, String errorCause) {
retryTemplate.execute(retryContext -> {
log.warn("Retrying {} time to set FAILED status for downloadJob userId: {} storageId: {}", retryContext.getRetryCount(), userId, downloadId);
downloadStatusPersistenceService.updateStatus(downloadId, DownloadStatusValue.FAILED, errorCause);
return null;
});
}

19
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/RedactionDlqMessageReceiver.java
View File

@ -1,11 +1,15 @@
package com.iqser.red.service.persistence.management.v1.processor.service.download;
import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.X_ERROR_INFO_HEADER;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionMessage;
import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DownloadStatusPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.settings.FileManagementServiceSettings;
import com.iqser.red.service.persistence.service.v1.api.shared.model.download.DownloadStatusValue;
import com.iqser.red.service.redaction.report.v1.api.model.ReportRequestMessage;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.experimental.FieldDefaults;
@ -32,6 +36,7 @@ public class RedactionDlqMessageReceiver {
RetryTemplate retryTemplate;
RabbitTemplate rabbitTemplate;
private final static int MAX_ERROR_CAUSE_LENGTH = 1024;
@RabbitHandler
@RabbitListener(queues = MessagingConfiguration.PDFTRON_DLQ)
@ -40,6 +45,14 @@ public class RedactionDlqMessageReceiver {
// Download packages will always be build on the pod that runs the scheduler, as we plan to replace the entire logic by downloading directory direct from storage, we leave it like that for now.
RedactionMessage redactionMessage = objectMapper.readValue(message.getBody(), RedactionMessage.class);
String errorCause = message.getMessageProperties().getHeader(X_ERROR_INFO_HEADER);
if (errorCause == null) {
errorCause = "Unkown error occured in pdftron service!";
}
else if(errorCause.length() > MAX_ERROR_CAUSE_LENGTH) {
errorCause = errorCause.substring(0,MAX_ERROR_CAUSE_LENGTH-1);
}
var fileEntryOptional = downloadPreparationService.getRedactionFileStatusEntry(redactionMessage.getDownloadId(), redactionMessage.getFileId());
@ -47,7 +60,7 @@ public class RedactionDlqMessageReceiver {
var entry = fileEntryOptional.get();
int numErrors = entry.getProcessingErrorCounter() + 1;
if (numErrors >= settings.getMaxRedactionFileErrorRetries()) {
setDownloadFailed(redactionMessage.getDownloadId());
setDownloadFailed(redactionMessage.getDownloadId(), errorCause);
downloadPreparationService.clearRedactionStatusEntries(redactionMessage.getDownloadId());
} else {
downloadPreparationService.increaseProcessingErrorCounter(redactionMessage, numErrors);
@ -60,11 +73,11 @@ public class RedactionDlqMessageReceiver {
}
public void setDownloadFailed(String downloadId) {
public void setDownloadFailed(String downloadId,String errorCause) {
retryTemplate.execute(retryContext -> {
log.warn("Retrying {} time to set FAILED status for downloadJob with storageId: {}", retryContext.getRetryCount(), downloadId);
downloadStatusPersistenceService.updateStatus(downloadId, DownloadStatusValue.FAILED);
downloadStatusPersistenceService.updateStatus(downloadId, DownloadStatusValue.FAILED, errorCause);
return null;
});
}

14
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DownloadStatusPersistenceService.java
View File

@ -7,8 +7,6 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
import jakarta.transaction.Transactional;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
@ -20,6 +18,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DownloadFileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.download.DownloadStatusValue;
import jakarta.transaction.Transactional;
import lombok.RequiredArgsConstructor;
@Service
@ -64,6 +63,13 @@ public class DownloadStatusPersistenceService {
}
@Transactional
public void updateStatus(String storageId, DownloadStatusValue status, String errorCause) {
downloadStatusRepository.updateStatusAndErrorCause(storageId, status, errorCause);
}
@Transactional
public void updateStatus(String storageId, DownloadStatusValue status) {
@ -75,6 +81,10 @@ public class DownloadStatusPersistenceService {
@Transactional
public void updateStatus(DownloadStatusEntity entity, DownloadStatusValue statusValue, long fileSize) {
if(statusValue.equals(DownloadStatusValue.READY)) {
entity.setErrorCause("");
}
entity.setStatus(statusValue);
entity.setFileSize(fileSize);
downloadStatusRepository.save(entity);

9
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/DownloadStatusRepository.java
View File

@ -16,16 +16,25 @@ public interface DownloadStatusRepository extends JpaRepository<DownloadStatusEn
List<DownloadStatusEntity> findAllByUserId(String userId);
@Modifying
@Query("update DownloadStatusEntity ds set ds.status = :status where ds.storageId = :storageId")
int updateStatus(@Param("storageId") String storageId, @Param("status") DownloadStatusValue status);
@Modifying
@Query("update DownloadStatusEntity ds set ds.status = :status, ds.errorCause = :errorCause where ds.storageId = :storageId")
int updateStatusAndErrorCause(@Param("storageId") String storageId, @Param("status") DownloadStatusValue status, @Param("errorCause") String errorCause);
@Modifying
@Transactional
@Query("update DownloadStatusEntity ds set ds.status = :status where ds.storageId = :storageId and ds.status != :status")
int updateStatusOnlyIfNotAlreadySet(@Param("storageId") String storageId, @Param("status") DownloadStatusValue status);
@Modifying
@Query("update DownloadStatusEntity ds set ds.lastDownload = :lastDownload where ds.storageId = :storageId")
void updateLastDownload(@Param("storageId") String storageId, @Param("lastDownload") OffsetDateTime lastDownload);
}

2
persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml
View File

@ -173,5 +173,7 @@ databaseChangeLog:
file: db/changelog/tenant/115-add-saas-migration-status-table.yaml
- include:
file: db/changelog/tenant/116-fix-null-fields-in-manual-redaction-table.yaml
- include:
file: db/changelog/tenant/117-add-download-status-error-cause.yaml
- include:
file: db/changelog/tenant/117-add-last-flag-calculation-date-to-file.yaml

11
persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/117-add-download-status-error-cause.yaml Normal file
View File

@ -0,0 +1,11 @@
databaseChangeLog:
- changeSet:
id: add-error-cause-to-download-status
author: yannik
changes:
- addColumn:
columns:
- column:
name: error_cause
type: VARCHAR(1024)
tableName: download_status

1
persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/download/DownloadStatus.java
View File

@ -21,6 +21,7 @@ public class DownloadStatus {
private String userId;
private String filename;
private String mimeType;
private String errorCause;
private DownloadStatusValue status;
private OffsetDateTime creationDate;
private OffsetDateTime lastDownload;