From 5bf42a6c60a82f76f1b1acc03c934bf528432cec Mon Sep 17 00:00:00 2001 From: devplant Date: Tue, 16 May 2023 18:55:59 +0300 Subject: [PATCH 1/6] RED-6072 - As Operation I want to see why files are in an ERROR state - add to file entity and models information about the error state - update endpoints with the error information in case of failure --- .../api/impl/controller/StatusController.java | 1 + ...tusProcessingUpdateInternalController.java | 22 +++++++---- .../FileStatusProcessingUpdateResource.java | 13 +++++-- .../processor/entity/dossier/FileEntity.java | 6 +++ .../service/CvAnalysisMessageReceiver.java | 4 +- .../FileStatusProcessingUpdateService.java | 37 +++++++++++++------ .../processor/service/FileStatusService.java | 4 +- .../service/ImageMessageReceiver.java | 5 ++- .../processor/service/NerMessageReceiver.java | 4 +- .../FileStatusPersistenceService.java | 11 ++++++ .../repository/FileRepository.java | 4 ++ .../v1/processor/utils/FileModelMapper.java | 3 +- .../db/changelog/db.changelog-tenant.yaml | 4 +- .../tenant/50-add-file-status-error-info.yaml | 17 +++++++++ .../integration/tests/FileProcessingTest.java | 17 ++++++--- .../v1/api/shared/model/FileStatus.java | 3 ++ .../dossier/file/FileErrorInfo.java | 17 +++++++++ .../dossier/file/FileModel.java | 1 + 18 files changed, 136 insertions(+), 37 deletions(-) create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/50-add-file-status-error-info.yaml create mode 100644 persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/FileErrorInfo.java diff --git a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/StatusController.java b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/StatusController.java index 93648210a..4d14dcced 100644 --- a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/StatusController.java +++ b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/StatusController.java @@ -486,6 +486,7 @@ public class StatusController implements StatusResource { .hasHighlights(status.isHasHighlights()) .lastIndexed(status.getLastIndexed()) .fileSize(status.getFileSize()) + .fileErrorInfo(status.getFileErrorInfo()) .build(); } diff --git a/persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/FileStatusProcessingUpdateInternalController.java b/persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/FileStatusProcessingUpdateInternalController.java index 59c917419..74ca10aad 100644 --- a/persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/FileStatusProcessingUpdateInternalController.java +++ b/persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/FileStatusProcessingUpdateInternalController.java @@ -2,12 +2,14 @@ package com.iqser.red.service.persistence.v1.internal.api.controller; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusProcessingUpdateService; import com.iqser.red.service.persistence.service.v1.api.internal.resources.FileStatusProcessingUpdateResource; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult; import com.iqser.red.service.persistence.service.v1.api.shared.model.UntouchedDocumentResponse; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -28,9 +30,10 @@ public class FileStatusProcessingUpdateInternalController implements FileStatusP } - public void preprocessingFailed(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId) { + public void preprocessingFailed(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId, + @RequestBody FileErrorInfo fileErrorInfo) { - fileStatusProcessingUpdateService.preprocessingFailed(dossierId, fileId); + fileStatusProcessingUpdateService.preprocessingFailed(dossierId, fileId, fileErrorInfo); } @@ -46,9 +49,10 @@ public class FileStatusProcessingUpdateInternalController implements FileStatusP } - public void analysisFailed(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId) { + public void analysisFailed(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId, + @RequestBody FileErrorInfo fileErrorInfo) { - fileStatusProcessingUpdateService.analysisFailed(dossierId, fileId); + fileStatusProcessingUpdateService.analysisFailed(dossierId, fileId, fileErrorInfo); } @@ -64,9 +68,10 @@ public class FileStatusProcessingUpdateInternalController implements FileStatusP } - public void ocrFailed(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId) { + public void ocrFailed(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId, + @RequestBody FileErrorInfo fileErrorInfo) { - fileStatusProcessingUpdateService.ocrFailed(dossierId, fileId); + fileStatusProcessingUpdateService.ocrFailed(dossierId, fileId, fileErrorInfo); } @@ -82,9 +87,10 @@ public class FileStatusProcessingUpdateInternalController implements FileStatusP } - public void indexingFailed(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId) { + public void indexingFailed(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId, + @RequestBody FileErrorInfo fileErrorInfo) { - fileStatusProcessingUpdateService.indexingFailed(dossierId, fileId); + fileStatusProcessingUpdateService.indexingFailed(dossierId, fileId, fileErrorInfo); } } diff --git a/persistence-service-v1/persistence-service-internal-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/internal/resources/FileStatusProcessingUpdateResource.java b/persistence-service-v1/persistence-service-internal-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/internal/resources/FileStatusProcessingUpdateResource.java index 8ceb301f4..9d832cceb 100644 --- a/persistence-service-v1/persistence-service-internal-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/internal/resources/FileStatusProcessingUpdateResource.java +++ b/persistence-service-v1/persistence-service-internal-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/internal/resources/FileStatusProcessingUpdateResource.java @@ -9,6 +9,7 @@ import org.springframework.web.bind.annotation.ResponseStatus; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult; import com.iqser.red.service.persistence.service.v1.api.shared.model.UntouchedDocumentResponse; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo; public interface FileStatusProcessingUpdateResource { @@ -29,7 +30,8 @@ public interface FileStatusProcessingUpdateResource { @ResponseStatus(value = HttpStatus.OK) @PostMapping(value = InternalApi.BASE_PATH + STATUS_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE + "/preprocessing-failed") - void preprocessingFailed(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId); + void preprocessingFailed(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId, + @RequestBody FileErrorInfo fileErrorInfo); @ResponseStatus(value = HttpStatus.OK) @@ -44,7 +46,8 @@ public interface FileStatusProcessingUpdateResource { @ResponseStatus(value = HttpStatus.OK) @PostMapping(value = InternalApi.BASE_PATH + STATUS_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE + "/analysis-failed") - void analysisFailed(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId); + void analysisFailed(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId, + @RequestBody FileErrorInfo fileErrorInfo); @ResponseStatus(value = HttpStatus.OK) @@ -59,7 +62,8 @@ public interface FileStatusProcessingUpdateResource { @ResponseStatus(value = HttpStatus.OK) @PostMapping(value = InternalApi.BASE_PATH + STATUS_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE + "/ocr-failed") - void ocrFailed(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId); + void ocrFailed(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId, + @RequestBody FileErrorInfo fileErrorInfo); @ResponseStatus(value = HttpStatus.OK) @@ -74,6 +78,7 @@ public interface FileStatusProcessingUpdateResource { @ResponseStatus(value = HttpStatus.OK) @PostMapping(value = InternalApi.BASE_PATH + STATUS_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE + "/indexing-failed") - void indexingFailed(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId); + void indexingFailed(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId, + @RequestBody FileErrorInfo fileErrorInfo); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java index 5e8ddf106..eac3ac018 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java @@ -180,6 +180,12 @@ public class FileEntity { @Column private Integer processingErrorCounter; + @Column + private String errorCause; + @Column + private String errorQueue; + @Column + private String errorService; public OffsetDateTime getLastOCRTime() { diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/CvAnalysisMessageReceiver.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/CvAnalysisMessageReceiver.java index 0548e910a..16964da0f 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/CvAnalysisMessageReceiver.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/CvAnalysisMessageReceiver.java @@ -7,6 +7,7 @@ import org.springframework.stereotype.Service; import com.fasterxml.jackson.databind.ObjectMapper; import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration; import com.iqser.red.service.persistence.management.v1.processor.model.CvAnalysisServiceResponse; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; @@ -40,9 +41,10 @@ public class CvAnalysisMessageReceiver { var response = objectMapper.readValue(failedMessage.getBody(), CvAnalysisServiceResponse.class); - fileStatusProcessingUpdateService.analysisFailed(response.getDossierId(), response.getFileId()); log.warn("Received message from {} for dossierId {} and fileId {}", MessagingConfiguration.IMAGE_SERVICE_DLQ, response.getDossierId(), response.getFileId()); + fileStatusProcessingUpdateService.analysisFailed(response.getDossierId(), response.getFileId(), new FileErrorInfo("cv analysis failed", MessagingConfiguration.IMAGE_SERVICE_DLQ, "cv-analysis-service-v1")); + } } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusProcessingUpdateService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusProcessingUpdateService.java index 666b13c7d..2a037417e 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusProcessingUpdateService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusProcessingUpdateService.java @@ -2,6 +2,7 @@ package com.iqser.red.service.persistence.management.v1.processor.service; import javax.transaction.Transactional; +import org.apache.commons.lang3.StringUtils; import org.springframework.retry.support.RetryTemplate; import org.springframework.web.bind.annotation.RestController; @@ -11,6 +12,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist import com.iqser.red.service.persistence.management.v1.processor.settings.FileManagementServiceSettings; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult; import com.iqser.red.service.persistence.service.v1.api.shared.model.UntouchedDocumentResponse; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus; import com.iqser.red.service.search.v1.model.IndexMessageType; @@ -101,17 +103,19 @@ public class FileStatusProcessingUpdateService { } - public void preprocessingFailed(String dossierId, String fileId) { - - setStatusError(dossierId, fileId, "preprocessingFailed"); + public void preprocessingFailed(String dossierId, String fileId, FileErrorInfo fileErrorInfo) { + if (fileErrorInfo != null && StringUtils.isEmpty(fileErrorInfo.getCause())) { + fileErrorInfo.setCause("preprocessingFailed"); + } + setStatusError(dossierId, fileId, fileErrorInfo); } - private void setStatusError(String dossierId, String fileId, String reason) { + private void setStatusError(String dossierId, String fileId, FileErrorInfo fileErrorInfo) { retryTemplate.execute(retryContext -> { - log.warn("Retrying {} time to set ERROR status for file {} in dossier {} with reason {} ", retryContext.getRetryCount(), fileId, dossierId, reason); - fileStatusService.setStatusError(fileId); + log.warn("Retrying {} time to set ERROR status for file {} in dossier {} with reason {} ", retryContext.getRetryCount(), fileId, dossierId, fileErrorInfo != null ? fileErrorInfo.getCause() : null); + fileStatusService.setStatusError(fileId, fileErrorInfo); return null; }); } @@ -150,15 +154,21 @@ public class FileStatusProcessingUpdateService { } - public void analysisFailed(String dossierId, String fileId) { + public void analysisFailed(String dossierId, String fileId, FileErrorInfo fileErrorInfo) { - setStatusError(dossierId, fileId, "analysisFailed"); + if (fileErrorInfo != null && StringUtils.isEmpty(fileErrorInfo.getCause())) { + fileErrorInfo.setCause("analysisFailed"); + } + setStatusError(dossierId, fileId, fileErrorInfo); } - public void ocrFailed(String dossierId, String fileId) { + public void ocrFailed(String dossierId, String fileId, FileErrorInfo fileErrorInfo) { - setStatusError(dossierId, fileId, "ocrFailed"); + if (fileErrorInfo != null && StringUtils.isEmpty(fileErrorInfo.getCause())) { + fileErrorInfo.setCause("ocrFailed"); + } + setStatusError(dossierId, fileId, fileErrorInfo); } @@ -182,9 +192,12 @@ public class FileStatusProcessingUpdateService { } - public void indexingFailed(String dossierId, String fileId) { + public void indexingFailed(String dossierId, String fileId, FileErrorInfo fileErrorInfo) { - setStatusError(dossierId, fileId, "indexingFailed"); + if (fileErrorInfo != null && StringUtils.isEmpty(fileErrorInfo.getCause())) { + fileErrorInfo.setCause("indexingFailed"); + } + setStatusError(dossierId, fileId, fileErrorInfo); } } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java index 3917100b7..dd4cf2409 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java @@ -43,6 +43,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequ import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; import com.iqser.red.service.persistence.service.v1.api.shared.model.MessageType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus; @@ -546,8 +547,9 @@ public class FileStatusService { } - public void setStatusError(String fileId) { + public void setStatusError(String fileId, FileErrorInfo fileErrorInfo) { + fileStatusPersistenceService.updateStatusErrorInfo(fileId, fileErrorInfo); fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.ERROR); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ImageMessageReceiver.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ImageMessageReceiver.java index adc404bba..a93582467 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ImageMessageReceiver.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ImageMessageReceiver.java @@ -12,6 +12,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration; import com.iqser.red.service.persistence.management.v1.processor.settings.FileManagementServiceSettings; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; import lombok.RequiredArgsConstructor; @@ -51,9 +52,9 @@ public class ImageMessageReceiver { String dossierId = (String) imageResponse.get("dossierId"); String fileId = (String) imageResponse.get("fileId"); - fileStatusProcessingUpdateService.analysisFailed(dossierId, fileId); - log.warn("Received message from {} for dossierId {} and fileId {}", MessagingConfiguration.IMAGE_SERVICE_DLQ, dossierId, fileId); + fileStatusProcessingUpdateService.analysisFailed(dossierId, fileId, new FileErrorInfo("image analysis failed", MessagingConfiguration.IMAGE_SERVICE_DLQ, "image-service-v3")); + } } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/NerMessageReceiver.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/NerMessageReceiver.java index 3715e9637..625ff0556 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/NerMessageReceiver.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/NerMessageReceiver.java @@ -10,6 +10,7 @@ import org.springframework.stereotype.Service; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; @@ -49,7 +50,8 @@ public class NerMessageReceiver { String fileId = (String) entityResponse.get("fileId"); log.warn("Received message {} for dossierId {} and fileId {}", MessagingConfiguration.NER_SERVICE_DLQ, dossierId, fileId); - fileStatusProcessingUpdateService.analysisFailed(dossierId, fileId); + fileStatusProcessingUpdateService.analysisFailed(dossierId, fileId, + new FileErrorInfo("ner service failed", MessagingConfiguration.NER_SERVICE_DLQ, "entity-recognition-service-v3")); } } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java index 6f01394a9..e75c96ccc 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java @@ -22,6 +22,7 @@ import com.iqser.red.service.persistence.management.v1.processor.model.OCRStatus import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileAttributesRepository; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus; @@ -163,6 +164,14 @@ public class FileStatusPersistenceService { } + public void updateStatusErrorInfo(String fileId, FileErrorInfo fileErrorInfo) { + if (fileErrorInfo == null) { + fileRepository.updateStatusErrorInfo(fileId, null, null, null); + } else { + fileRepository.updateStatusErrorInfo(fileId, fileErrorInfo.getCause(), fileErrorInfo.getQueue(), fileErrorInfo.getService()); + } + + } @Transactional public void updateProcessingStatus(String fileId, ProcessingStatus processingStatus) { @@ -170,6 +179,8 @@ public class FileStatusPersistenceService { return; } if (processingStatus == ProcessingStatus.PROCESSED) { + // reset the error info + fileRepository.updateStatusErrorInfo(fileId, null, null, null); // In case the file is updated to "processed", "lastProcessed" date should be updated to "now" fileRepository.updateProcessingStatus(fileId, processingStatus, diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java index 61ecef7f4..9880fc7bb 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java @@ -81,6 +81,10 @@ public interface FileRepository extends JpaRepository { @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated, f.processingErrorCounter = :processingErrorCounter " + "where f.id = :fileId") void updateProcessingStatus(String fileId, ProcessingStatus processingStatus, OffsetDateTime lastUpdated, int processingErrorCounter); + @Modifying(clearAutomatically = true) + @Query("update FileEntity f set f.errorCause = :cause, f.errorQueue = :queue, f.errorService = :service " + "where f.id = :fileId") + void updateStatusErrorInfo(String fileId, String cause, String queue, String service); + @Modifying(clearAutomatically = true, flushAutomatically = true) @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated, f.lastProcessed = :lastProcessed, f.processingErrorCounter = :processingErrorCounter " + "where f.id = :fileId") diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/FileModelMapper.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/FileModelMapper.java index e01823312..4d071cb41 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/FileModelMapper.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/FileModelMapper.java @@ -3,6 +3,7 @@ package com.iqser.red.service.persistence.management.v1.processor.utils; import java.util.function.BiConsumer; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel; public class FileModelMapper implements BiConsumer { @@ -11,7 +12,7 @@ public class FileModelMapper implements BiConsumer { public void accept(FileEntity fileEntity, FileModel fileModel) { fileEntity.getFileAttributes().forEach(fa -> fileModel.getFileAttributes().put(fa.getFileAttributeId().getFileAttributeConfigId(), fa.getValue())); - + fileModel.setFileErrorInfo(new FileErrorInfo(fileEntity.getErrorCause(), fileEntity.getErrorQueue(), fileEntity.getErrorService())); } } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml index e807ab9d6..1a2cfd855 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml @@ -132,4 +132,6 @@ databaseChangeLog: - include: file: db/changelog/tenant/48-add-watermark-text-alignment.yaml - include: - file: db/changelog/tenant/sql/49-add-keep_overlapping_objects.sql \ No newline at end of file + file: db/changelog/tenant/sql/49-add-keep_overlapping_objects.sql + - include: + file: db/changelog/tenant/50-add-file-status-error-info.yaml diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/50-add-file-status-error-info.yaml b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/50-add-file-status-error-info.yaml new file mode 100644 index 000000000..9ea4cc1c2 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/50-add-file-status-error-info.yaml @@ -0,0 +1,17 @@ +databaseChangeLog: + - changeSet: + id: add-file-status-error-info + author: corinaolariu + changes: + - addColumn: + columns: + - column: + name: error_cause + type: VARCHAR(255) + - column: + name: error_queue + type: VARCHAR(255) + - column: + name: error_service + type: VARCHAR(255) + tableName: file \ No newline at end of file diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileProcessingTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileProcessingTest.java index 669566b3b..7fea05015 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileProcessingTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileProcessingTest.java @@ -19,6 +19,7 @@ import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPers import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult; import com.iqser.red.service.persistence.service.v1.api.shared.model.MessageType; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus; @@ -57,7 +58,7 @@ public class FileProcessingTest extends AbstractPersistenceServerServiceTest { assertThat(file.getOcrStartTime()).isNull(); - fileProcessingClient.analysisFailed(dossier.getId(), file.getId()); + fileProcessingClient.analysisFailed(dossier.getId(), file.getId(), new FileErrorInfo()); var loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.ERROR); @@ -68,7 +69,7 @@ public class FileProcessingTest extends AbstractPersistenceServerServiceTest { assertThat(loadedFile.getWorkflowStatus()).isEqualTo(WorkflowStatus.NEW); assertThat(loadedFile.getAnalysisVersion()).isEqualTo(0); - fileProcessingClient.ocrFailed(dossier.getId(), file.getId()); + fileProcessingClient.ocrFailed(dossier.getId(), file.getId(), null); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.ERROR); @@ -81,7 +82,7 @@ public class FileProcessingTest extends AbstractPersistenceServerServiceTest { assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.INDEXING); assertThat(loadedFile.getLastIndexed()).isNull(); - fileProcessingClient.indexingFailed(dossier.getId(), file.getId()); + fileProcessingClient.indexingFailed(dossier.getId(), file.getId(), null); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.ERROR); @@ -102,9 +103,11 @@ public class FileProcessingTest extends AbstractPersistenceServerServiceTest { assertThat(file.getOcrStartTime()).isNull(); - fileProcessingClient.analysisFailed(dossier.getId(), file.getId()); + var queue = "queue1"; + fileProcessingClient.analysisFailed(dossier.getId(), file.getId(), new FileErrorInfo("something failed", queue, "service")); var loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.ERROR); + assertThat(loadedFile.getFileErrorInfo().getQueue()).isEqualTo(queue); fileProcessingClient.analysisSuccessful(dossier.getId(), file.getId(), @@ -129,7 +132,7 @@ public class FileProcessingTest extends AbstractPersistenceServerServiceTest { // assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.DELETED); assertThat(loadedFile.getSoftDeletedTime()).isNotNull(); - fileProcessingClient.indexingFailed(dossier.getId(), file.getId()); + fileProcessingClient.indexingFailed(dossier.getId(), file.getId(), null); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); // assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.DELETED); assertThat(loadedFile.getSoftDeletedTime()).isNotNull(); @@ -202,9 +205,11 @@ public class FileProcessingTest extends AbstractPersistenceServerServiceTest { var file = fileTesterAndProvider.testAndProvideFile(dossier); - fileProcessingClient.analysisFailed(dossier.getId(), file.getId()); + fileProcessingClient.analysisFailed(dossier.getId(), file.getId(), null); var loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.ERROR); + assertThat(loadedFile.getFileErrorInfo()).isNotNull(); + assertThat(loadedFile.getFileErrorInfo().getCause()).isNull(); reanalysisClient.reanalyzeFilesForDossier(dossier.getId(), List.of(loadedFile.getId()), true); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FileStatus.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FileStatus.java index 3b4af8ea6..894f85630 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FileStatus.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FileStatus.java @@ -3,6 +3,7 @@ package com.iqser.red.service.persistence.service.v1.api.shared.model; import java.time.OffsetDateTime; import java.util.Set; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus; @@ -138,6 +139,8 @@ public class FileStatus { private int analysisVersion; @Schema(description = "Last time the file was indexed in ES.") private OffsetDateTime lastIndexed; + @Schema(description = "The error information for the error state of the file") + private FileErrorInfo fileErrorInfo; @Schema(description = "Shows if this file has been OCRed by us. Last Time of OCR.") public OffsetDateTime getLastOCRTime() { diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/FileErrorInfo.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/FileErrorInfo.java new file mode 100644 index 000000000..0df2b7d95 --- /dev/null +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/FileErrorInfo.java @@ -0,0 +1,17 @@ +package com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@AllArgsConstructor +@NoArgsConstructor +public class FileErrorInfo { + + private String cause; + private String queue; + private String service; +} diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/FileModel.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/FileModel.java index 06158291e..70156257a 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/FileModel.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/FileModel.java @@ -70,6 +70,7 @@ public class FileModel { private OffsetDateTime redactionModificationDate; private OffsetDateTime fileManipulationDate; private boolean hasHighlights; + private FileErrorInfo fileErrorInfo; public boolean isAnalysisRequired() { From 1858e462e0d531eef9be06b60fa562c4a11c4d4f Mon Sep 17 00:00:00 2001 From: devplant Date: Wed, 17 May 2023 10:10:51 +0300 Subject: [PATCH 2/6] RED-6072 - As Operation I want to see why files are in an ERROR state - update junit test --- .../v1/server/integration/tests/FileProcessingTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileProcessingTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileProcessingTest.java index 7fea05015..226bbda24 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileProcessingTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileProcessingTest.java @@ -205,7 +205,7 @@ public class FileProcessingTest extends AbstractPersistenceServerServiceTest { var file = fileTesterAndProvider.testAndProvideFile(dossier); - fileProcessingClient.analysisFailed(dossier.getId(), file.getId(), null); + fileProcessingClient.analysisFailed(dossier.getId(), file.getId(), new FileErrorInfo()); var loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.ERROR); assertThat(loadedFile.getFileErrorInfo()).isNotNull(); From 43e7b0a3eb33830d1b351036fa59fc1b16064f32 Mon Sep 17 00:00:00 2001 From: devplant Date: Wed, 17 May 2023 15:32:41 +0300 Subject: [PATCH 3/6] RED-6072 - As Operation I want to see why files are in an ERROR state - set back the initial endpoints for backward compatibility --- ...tusProcessingUpdateInternalController.java | 27 +++++++++++++++++++ .../FileStatusProcessingUpdateResource.java | 22 +++++++++++++++ .../repository/FileRepository.java | 4 +-- 3 files changed, 51 insertions(+), 2 deletions(-) diff --git a/persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/FileStatusProcessingUpdateInternalController.java b/persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/FileStatusProcessingUpdateInternalController.java index 74ca10aad..634f759c0 100644 --- a/persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/FileStatusProcessingUpdateInternalController.java +++ b/persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/FileStatusProcessingUpdateInternalController.java @@ -5,6 +5,7 @@ import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; +import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration; import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusProcessingUpdateService; import com.iqser.red.service.persistence.service.v1.api.internal.resources.FileStatusProcessingUpdateResource; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult; @@ -30,6 +31,13 @@ public class FileStatusProcessingUpdateInternalController implements FileStatusP } + public void preprocessingFailed(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId) { + + fileStatusProcessingUpdateService.preprocessingFailed(dossierId, fileId, new FileErrorInfo("preprocessing failed", + MessagingConfiguration.PRE_PROCESSING_DLQ, "pdftron-service")); + } + + public void preprocessingFailed(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId, @RequestBody FileErrorInfo fileErrorInfo) { @@ -49,6 +57,12 @@ public class FileStatusProcessingUpdateInternalController implements FileStatusP } + public void analysisFailed(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId) { + + fileStatusProcessingUpdateService.analysisFailed(dossierId, fileId, new FileErrorInfo()); + } + + public void analysisFailed(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId, @RequestBody FileErrorInfo fileErrorInfo) { @@ -68,6 +82,13 @@ public class FileStatusProcessingUpdateInternalController implements FileStatusP } + public void ocrFailed(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId) { + + fileStatusProcessingUpdateService.ocrFailed(dossierId, fileId, new FileErrorInfo("ocr failed", + MessagingConfiguration.OCR_DLQ, "ocr-service")); + } + + public void ocrFailed(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId, @RequestBody FileErrorInfo fileErrorInfo) { @@ -87,6 +108,12 @@ public class FileStatusProcessingUpdateInternalController implements FileStatusP } + public void indexingFailed(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId) { + + fileStatusProcessingUpdateService.indexingFailed(dossierId, fileId, new FileErrorInfo("indexing failed", + MessagingConfiguration.INDEXING_DQL, "search-service")); + } + public void indexingFailed(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId, @RequestBody FileErrorInfo fileErrorInfo) { diff --git a/persistence-service-v1/persistence-service-internal-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/internal/resources/FileStatusProcessingUpdateResource.java b/persistence-service-v1/persistence-service-internal-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/internal/resources/FileStatusProcessingUpdateResource.java index 9d832cceb..5bf962ea0 100644 --- a/persistence-service-v1/persistence-service-internal-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/internal/resources/FileStatusProcessingUpdateResource.java +++ b/persistence-service-v1/persistence-service-internal-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/internal/resources/FileStatusProcessingUpdateResource.java @@ -27,6 +27,10 @@ public interface FileStatusProcessingUpdateResource { @PathVariable(FILE_ID) String fileId, @RequestBody UntouchedDocumentResponse untouchedDocumentResponse); + @Deprecated + @ResponseStatus(value = HttpStatus.OK) + @PostMapping(value = InternalApi.BASE_PATH + STATUS_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE + "/preprocessing-failed") + void preprocessingFailed(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId); @ResponseStatus(value = HttpStatus.OK) @PostMapping(value = InternalApi.BASE_PATH + STATUS_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE + "/preprocessing-failed") @@ -44,6 +48,12 @@ public interface FileStatusProcessingUpdateResource { void ocrSuccessful(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId); + @Deprecated + @ResponseStatus(value = HttpStatus.OK) + @PostMapping(value = InternalApi.BASE_PATH + STATUS_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE + "/analysis-failed") + void analysisFailed(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId); + + @ResponseStatus(value = HttpStatus.OK) @PostMapping(value = InternalApi.BASE_PATH + STATUS_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE + "/analysis-failed") void analysisFailed(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId, @@ -60,6 +70,12 @@ public interface FileStatusProcessingUpdateResource { void ocrProcessing(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId); + @Deprecated + @ResponseStatus(value = HttpStatus.OK) + @PostMapping(value = InternalApi.BASE_PATH + STATUS_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE + "/ocr-failed") + void ocrFailed(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId); + + @ResponseStatus(value = HttpStatus.OK) @PostMapping(value = InternalApi.BASE_PATH + STATUS_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE + "/ocr-failed") void ocrFailed(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId, @@ -76,6 +92,12 @@ public interface FileStatusProcessingUpdateResource { void indexingSuccessful(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId); + @Deprecated + @ResponseStatus(value = HttpStatus.OK) + @PostMapping(value = InternalApi.BASE_PATH + STATUS_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE + "/indexing-failed") + void indexingFailed(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId); + + @ResponseStatus(value = HttpStatus.OK) @PostMapping(value = InternalApi.BASE_PATH + STATUS_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE + "/indexing-failed") void indexingFailed(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId, diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java index 9880fc7bb..f1e543b50 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java @@ -81,8 +81,8 @@ public interface FileRepository extends JpaRepository { @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated, f.processingErrorCounter = :processingErrorCounter " + "where f.id = :fileId") void updateProcessingStatus(String fileId, ProcessingStatus processingStatus, OffsetDateTime lastUpdated, int processingErrorCounter); - @Modifying(clearAutomatically = true) - @Query("update FileEntity f set f.errorCause = :cause, f.errorQueue = :queue, f.errorService = :service " + "where f.id = :fileId") + @Modifying + @Query("update FileEntity f set f.errorCause = :cause, f.errorQueue = :queue, f.errorService = :service where f.id = :fileId") void updateStatusErrorInfo(String fileId, String cause, String queue, String service); From b1d1c41dbfc7f956ba23628c69d1b13a4e648f02 Mon Sep 17 00:00:00 2001 From: devplant Date: Thu, 18 May 2023 08:57:06 +0300 Subject: [PATCH 4/6] RED-6072 - As Operation I want to see why files are in an ERROR state - update the new endpoints with different path than the original endpoints --- .../resources/FileStatusProcessingUpdateResource.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/persistence-service-v1/persistence-service-internal-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/internal/resources/FileStatusProcessingUpdateResource.java b/persistence-service-v1/persistence-service-internal-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/internal/resources/FileStatusProcessingUpdateResource.java index 5bf962ea0..4b1b4fbf5 100644 --- a/persistence-service-v1/persistence-service-internal-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/internal/resources/FileStatusProcessingUpdateResource.java +++ b/persistence-service-v1/persistence-service-internal-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/internal/resources/FileStatusProcessingUpdateResource.java @@ -33,7 +33,7 @@ public interface FileStatusProcessingUpdateResource { void preprocessingFailed(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId); @ResponseStatus(value = HttpStatus.OK) - @PostMapping(value = InternalApi.BASE_PATH + STATUS_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE + "/preprocessing-failed") + @PostMapping(value = InternalApi.BASE_PATH + STATUS_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE + "/preprocessing-failed-error") void preprocessingFailed(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId, @RequestBody FileErrorInfo fileErrorInfo); @@ -55,7 +55,7 @@ public interface FileStatusProcessingUpdateResource { @ResponseStatus(value = HttpStatus.OK) - @PostMapping(value = InternalApi.BASE_PATH + STATUS_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE + "/analysis-failed") + @PostMapping(value = InternalApi.BASE_PATH + STATUS_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE + "/analysis-failed-error") void analysisFailed(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId, @RequestBody FileErrorInfo fileErrorInfo); @@ -77,7 +77,7 @@ public interface FileStatusProcessingUpdateResource { @ResponseStatus(value = HttpStatus.OK) - @PostMapping(value = InternalApi.BASE_PATH + STATUS_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE + "/ocr-failed") + @PostMapping(value = InternalApi.BASE_PATH + STATUS_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE + "/ocr-failed-error") void ocrFailed(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId, @RequestBody FileErrorInfo fileErrorInfo); @@ -99,7 +99,7 @@ public interface FileStatusProcessingUpdateResource { @ResponseStatus(value = HttpStatus.OK) - @PostMapping(value = InternalApi.BASE_PATH + STATUS_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE + "/indexing-failed") + @PostMapping(value = InternalApi.BASE_PATH + STATUS_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE + "/indexing-failed-error") void indexingFailed(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId, @RequestBody FileErrorInfo fileErrorInfo); From 54bfe80f3167341f3386b81dfb3f45d4932a7bab Mon Sep 17 00:00:00 2001 From: devplant Date: Thu, 18 May 2023 10:05:14 +0300 Subject: [PATCH 5/6] RED-6072 - As Operation I want to see why files are in an ERROR state - add @Transactional for the update --- .../service/persistence/FileStatusPersistenceService.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java index e75c96ccc..ab7295a85 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java @@ -164,6 +164,7 @@ public class FileStatusPersistenceService { } + @Transactional public void updateStatusErrorInfo(String fileId, FileErrorInfo fileErrorInfo) { if (fileErrorInfo == null) { fileRepository.updateStatusErrorInfo(fileId, null, null, null); @@ -172,6 +173,7 @@ public class FileStatusPersistenceService { } } + @Transactional public void updateProcessingStatus(String fileId, ProcessingStatus processingStatus) { From eff8eeaf03f76c12ca700ff4bb4f7688c291fab6 Mon Sep 17 00:00:00 2001 From: devplant Date: Thu, 18 May 2023 10:50:49 +0300 Subject: [PATCH 6/6] RED-6072 - As Operation I want to see why files are in an ERROR state - update junit tests --- .../v1/server/integration/tests/FileProcessingTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileProcessingTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileProcessingTest.java index 226bbda24..905befc26 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileProcessingTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileProcessingTest.java @@ -69,7 +69,7 @@ public class FileProcessingTest extends AbstractPersistenceServerServiceTest { assertThat(loadedFile.getWorkflowStatus()).isEqualTo(WorkflowStatus.NEW); assertThat(loadedFile.getAnalysisVersion()).isEqualTo(0); - fileProcessingClient.ocrFailed(dossier.getId(), file.getId(), null); + fileProcessingClient.ocrFailed(dossier.getId(), file.getId(), new FileErrorInfo()); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.ERROR); @@ -82,7 +82,7 @@ public class FileProcessingTest extends AbstractPersistenceServerServiceTest { assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.INDEXING); assertThat(loadedFile.getLastIndexed()).isNull(); - fileProcessingClient.indexingFailed(dossier.getId(), file.getId(), null); + fileProcessingClient.indexingFailed(dossier.getId(), file.getId(), new FileErrorInfo()); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.ERROR); @@ -132,7 +132,7 @@ public class FileProcessingTest extends AbstractPersistenceServerServiceTest { // assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.DELETED); assertThat(loadedFile.getSoftDeletedTime()).isNotNull(); - fileProcessingClient.indexingFailed(dossier.getId(), file.getId(), null); + fileProcessingClient.indexingFailed(dossier.getId(), file.getId(), new FileErrorInfo()); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); // assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.DELETED); assertThat(loadedFile.getSoftDeletedTime()).isNotNull(); @@ -209,7 +209,7 @@ public class FileProcessingTest extends AbstractPersistenceServerServiceTest { var loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.ERROR); assertThat(loadedFile.getFileErrorInfo()).isNotNull(); - assertThat(loadedFile.getFileErrorInfo().getCause()).isNull(); + assertThat(loadedFile.getFileErrorInfo().getCause()).isEqualTo("analysisFailed"); reanalysisClient.reanalyzeFilesForDossier(dossier.getId(), List.of(loadedFile.getId()), true); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());