diff --git a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/StatusController.java b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/StatusController.java index 93648210a..4d14dcced 100644 --- a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/StatusController.java +++ b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/StatusController.java @@ -486,6 +486,7 @@ public class StatusController implements StatusResource { .hasHighlights(status.isHasHighlights()) .lastIndexed(status.getLastIndexed()) .fileSize(status.getFileSize()) + .fileErrorInfo(status.getFileErrorInfo()) .build(); } diff --git a/persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/FileStatusProcessingUpdateInternalController.java b/persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/FileStatusProcessingUpdateInternalController.java index 59c917419..634f759c0 100644 --- a/persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/FileStatusProcessingUpdateInternalController.java +++ b/persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/FileStatusProcessingUpdateInternalController.java @@ -2,12 +2,15 @@ package com.iqser.red.service.persistence.v1.internal.api.controller; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; +import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration; import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusProcessingUpdateService; import com.iqser.red.service.persistence.service.v1.api.internal.resources.FileStatusProcessingUpdateResource; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult; import com.iqser.red.service.persistence.service.v1.api.shared.model.UntouchedDocumentResponse; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -30,7 +33,15 @@ public class FileStatusProcessingUpdateInternalController implements FileStatusP public void preprocessingFailed(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId) { - fileStatusProcessingUpdateService.preprocessingFailed(dossierId, fileId); + fileStatusProcessingUpdateService.preprocessingFailed(dossierId, fileId, new FileErrorInfo("preprocessing failed", + MessagingConfiguration.PRE_PROCESSING_DLQ, "pdftron-service")); + } + + + public void preprocessingFailed(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId, + @RequestBody FileErrorInfo fileErrorInfo) { + + fileStatusProcessingUpdateService.preprocessingFailed(dossierId, fileId, fileErrorInfo); } @@ -48,7 +59,14 @@ public class FileStatusProcessingUpdateInternalController implements FileStatusP public void analysisFailed(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId) { - fileStatusProcessingUpdateService.analysisFailed(dossierId, fileId); + fileStatusProcessingUpdateService.analysisFailed(dossierId, fileId, new FileErrorInfo()); + } + + + public void analysisFailed(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId, + @RequestBody FileErrorInfo fileErrorInfo) { + + fileStatusProcessingUpdateService.analysisFailed(dossierId, fileId, fileErrorInfo); } @@ -66,7 +84,15 @@ public class FileStatusProcessingUpdateInternalController implements FileStatusP public void ocrFailed(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId) { - fileStatusProcessingUpdateService.ocrFailed(dossierId, fileId); + fileStatusProcessingUpdateService.ocrFailed(dossierId, fileId, new FileErrorInfo("ocr failed", + MessagingConfiguration.OCR_DLQ, "ocr-service")); + } + + + public void ocrFailed(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId, + @RequestBody FileErrorInfo fileErrorInfo) { + + fileStatusProcessingUpdateService.ocrFailed(dossierId, fileId, fileErrorInfo); } @@ -84,7 +110,14 @@ public class FileStatusProcessingUpdateInternalController implements FileStatusP public void indexingFailed(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId) { - fileStatusProcessingUpdateService.indexingFailed(dossierId, fileId); + fileStatusProcessingUpdateService.indexingFailed(dossierId, fileId, new FileErrorInfo("indexing failed", + MessagingConfiguration.INDEXING_DQL, "search-service")); + } + + public void indexingFailed(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId, + @RequestBody FileErrorInfo fileErrorInfo) { + + fileStatusProcessingUpdateService.indexingFailed(dossierId, fileId, fileErrorInfo); } } diff --git a/persistence-service-v1/persistence-service-internal-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/internal/resources/FileStatusProcessingUpdateResource.java b/persistence-service-v1/persistence-service-internal-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/internal/resources/FileStatusProcessingUpdateResource.java index 8ceb301f4..4b1b4fbf5 100644 --- a/persistence-service-v1/persistence-service-internal-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/internal/resources/FileStatusProcessingUpdateResource.java +++ b/persistence-service-v1/persistence-service-internal-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/internal/resources/FileStatusProcessingUpdateResource.java @@ -9,6 +9,7 @@ import org.springframework.web.bind.annotation.ResponseStatus; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult; import com.iqser.red.service.persistence.service.v1.api.shared.model.UntouchedDocumentResponse; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo; public interface FileStatusProcessingUpdateResource { @@ -26,11 +27,16 @@ public interface FileStatusProcessingUpdateResource { @PathVariable(FILE_ID) String fileId, @RequestBody UntouchedDocumentResponse untouchedDocumentResponse); - + @Deprecated @ResponseStatus(value = HttpStatus.OK) @PostMapping(value = InternalApi.BASE_PATH + STATUS_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE + "/preprocessing-failed") void preprocessingFailed(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId); + @ResponseStatus(value = HttpStatus.OK) + @PostMapping(value = InternalApi.BASE_PATH + STATUS_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE + "/preprocessing-failed-error") + void preprocessingFailed(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId, + @RequestBody FileErrorInfo fileErrorInfo); + @ResponseStatus(value = HttpStatus.OK) @PostMapping(value = InternalApi.BASE_PATH + STATUS_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE + "/analysis-successful", consumes = MediaType.APPLICATION_JSON_VALUE) @@ -42,11 +48,18 @@ public interface FileStatusProcessingUpdateResource { void ocrSuccessful(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId); + @Deprecated @ResponseStatus(value = HttpStatus.OK) @PostMapping(value = InternalApi.BASE_PATH + STATUS_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE + "/analysis-failed") void analysisFailed(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId); + @ResponseStatus(value = HttpStatus.OK) + @PostMapping(value = InternalApi.BASE_PATH + STATUS_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE + "/analysis-failed-error") + void analysisFailed(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId, + @RequestBody FileErrorInfo fileErrorInfo); + + @ResponseStatus(value = HttpStatus.OK) @PostMapping(value = InternalApi.BASE_PATH + STATUS_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE + "/pre-processing") void preProcessing(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId); @@ -57,11 +70,18 @@ public interface FileStatusProcessingUpdateResource { void ocrProcessing(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId); + @Deprecated @ResponseStatus(value = HttpStatus.OK) @PostMapping(value = InternalApi.BASE_PATH + STATUS_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE + "/ocr-failed") void ocrFailed(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId); + @ResponseStatus(value = HttpStatus.OK) + @PostMapping(value = InternalApi.BASE_PATH + STATUS_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE + "/ocr-failed-error") + void ocrFailed(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId, + @RequestBody FileErrorInfo fileErrorInfo); + + @ResponseStatus(value = HttpStatus.OK) @PostMapping(value = InternalApi.BASE_PATH + STATUS_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE + "/indexing") void indexing(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId); @@ -72,8 +92,15 @@ public interface FileStatusProcessingUpdateResource { void indexingSuccessful(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId); + @Deprecated @ResponseStatus(value = HttpStatus.OK) @PostMapping(value = InternalApi.BASE_PATH + STATUS_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE + "/indexing-failed") void indexingFailed(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId); + + @ResponseStatus(value = HttpStatus.OK) + @PostMapping(value = InternalApi.BASE_PATH + STATUS_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE + "/indexing-failed-error") + void indexingFailed(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId, + @RequestBody FileErrorInfo fileErrorInfo); + } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java index 5e8ddf106..eac3ac018 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java @@ -180,6 +180,12 @@ public class FileEntity { @Column private Integer processingErrorCounter; + @Column + private String errorCause; + @Column + private String errorQueue; + @Column + private String errorService; public OffsetDateTime getLastOCRTime() { diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/CvAnalysisMessageReceiver.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/CvAnalysisMessageReceiver.java index 0548e910a..16964da0f 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/CvAnalysisMessageReceiver.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/CvAnalysisMessageReceiver.java @@ -7,6 +7,7 @@ import org.springframework.stereotype.Service; import com.fasterxml.jackson.databind.ObjectMapper; import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration; import com.iqser.red.service.persistence.management.v1.processor.model.CvAnalysisServiceResponse; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; @@ -40,9 +41,10 @@ public class CvAnalysisMessageReceiver { var response = objectMapper.readValue(failedMessage.getBody(), CvAnalysisServiceResponse.class); - fileStatusProcessingUpdateService.analysisFailed(response.getDossierId(), response.getFileId()); log.warn("Received message from {} for dossierId {} and fileId {}", MessagingConfiguration.IMAGE_SERVICE_DLQ, response.getDossierId(), response.getFileId()); + fileStatusProcessingUpdateService.analysisFailed(response.getDossierId(), response.getFileId(), new FileErrorInfo("cv analysis failed", MessagingConfiguration.IMAGE_SERVICE_DLQ, "cv-analysis-service-v1")); + } } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusProcessingUpdateService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusProcessingUpdateService.java index 666b13c7d..2a037417e 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusProcessingUpdateService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusProcessingUpdateService.java @@ -2,6 +2,7 @@ package com.iqser.red.service.persistence.management.v1.processor.service; import javax.transaction.Transactional; +import org.apache.commons.lang3.StringUtils; import org.springframework.retry.support.RetryTemplate; import org.springframework.web.bind.annotation.RestController; @@ -11,6 +12,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist import com.iqser.red.service.persistence.management.v1.processor.settings.FileManagementServiceSettings; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult; import com.iqser.red.service.persistence.service.v1.api.shared.model.UntouchedDocumentResponse; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus; import com.iqser.red.service.search.v1.model.IndexMessageType; @@ -101,17 +103,19 @@ public class FileStatusProcessingUpdateService { } - public void preprocessingFailed(String dossierId, String fileId) { - - setStatusError(dossierId, fileId, "preprocessingFailed"); + public void preprocessingFailed(String dossierId, String fileId, FileErrorInfo fileErrorInfo) { + if (fileErrorInfo != null && StringUtils.isEmpty(fileErrorInfo.getCause())) { + fileErrorInfo.setCause("preprocessingFailed"); + } + setStatusError(dossierId, fileId, fileErrorInfo); } - private void setStatusError(String dossierId, String fileId, String reason) { + private void setStatusError(String dossierId, String fileId, FileErrorInfo fileErrorInfo) { retryTemplate.execute(retryContext -> { - log.warn("Retrying {} time to set ERROR status for file {} in dossier {} with reason {} ", retryContext.getRetryCount(), fileId, dossierId, reason); - fileStatusService.setStatusError(fileId); + log.warn("Retrying {} time to set ERROR status for file {} in dossier {} with reason {} ", retryContext.getRetryCount(), fileId, dossierId, fileErrorInfo != null ? fileErrorInfo.getCause() : null); + fileStatusService.setStatusError(fileId, fileErrorInfo); return null; }); } @@ -150,15 +154,21 @@ public class FileStatusProcessingUpdateService { } - public void analysisFailed(String dossierId, String fileId) { + public void analysisFailed(String dossierId, String fileId, FileErrorInfo fileErrorInfo) { - setStatusError(dossierId, fileId, "analysisFailed"); + if (fileErrorInfo != null && StringUtils.isEmpty(fileErrorInfo.getCause())) { + fileErrorInfo.setCause("analysisFailed"); + } + setStatusError(dossierId, fileId, fileErrorInfo); } - public void ocrFailed(String dossierId, String fileId) { + public void ocrFailed(String dossierId, String fileId, FileErrorInfo fileErrorInfo) { - setStatusError(dossierId, fileId, "ocrFailed"); + if (fileErrorInfo != null && StringUtils.isEmpty(fileErrorInfo.getCause())) { + fileErrorInfo.setCause("ocrFailed"); + } + setStatusError(dossierId, fileId, fileErrorInfo); } @@ -182,9 +192,12 @@ public class FileStatusProcessingUpdateService { } - public void indexingFailed(String dossierId, String fileId) { + public void indexingFailed(String dossierId, String fileId, FileErrorInfo fileErrorInfo) { - setStatusError(dossierId, fileId, "indexingFailed"); + if (fileErrorInfo != null && StringUtils.isEmpty(fileErrorInfo.getCause())) { + fileErrorInfo.setCause("indexingFailed"); + } + setStatusError(dossierId, fileId, fileErrorInfo); } } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java index 3917100b7..dd4cf2409 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java @@ -43,6 +43,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequ import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; import com.iqser.red.service.persistence.service.v1.api.shared.model.MessageType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus; @@ -546,8 +547,9 @@ public class FileStatusService { } - public void setStatusError(String fileId) { + public void setStatusError(String fileId, FileErrorInfo fileErrorInfo) { + fileStatusPersistenceService.updateStatusErrorInfo(fileId, fileErrorInfo); fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.ERROR); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ImageMessageReceiver.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ImageMessageReceiver.java index adc404bba..a93582467 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ImageMessageReceiver.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ImageMessageReceiver.java @@ -12,6 +12,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration; import com.iqser.red.service.persistence.management.v1.processor.settings.FileManagementServiceSettings; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; import lombok.RequiredArgsConstructor; @@ -51,9 +52,9 @@ public class ImageMessageReceiver { String dossierId = (String) imageResponse.get("dossierId"); String fileId = (String) imageResponse.get("fileId"); - fileStatusProcessingUpdateService.analysisFailed(dossierId, fileId); - log.warn("Received message from {} for dossierId {} and fileId {}", MessagingConfiguration.IMAGE_SERVICE_DLQ, dossierId, fileId); + fileStatusProcessingUpdateService.analysisFailed(dossierId, fileId, new FileErrorInfo("image analysis failed", MessagingConfiguration.IMAGE_SERVICE_DLQ, "image-service-v3")); + } } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/NerMessageReceiver.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/NerMessageReceiver.java index 3715e9637..625ff0556 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/NerMessageReceiver.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/NerMessageReceiver.java @@ -10,6 +10,7 @@ import org.springframework.stereotype.Service; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; @@ -49,7 +50,8 @@ public class NerMessageReceiver { String fileId = (String) entityResponse.get("fileId"); log.warn("Received message {} for dossierId {} and fileId {}", MessagingConfiguration.NER_SERVICE_DLQ, dossierId, fileId); - fileStatusProcessingUpdateService.analysisFailed(dossierId, fileId); + fileStatusProcessingUpdateService.analysisFailed(dossierId, fileId, + new FileErrorInfo("ner service failed", MessagingConfiguration.NER_SERVICE_DLQ, "entity-recognition-service-v3")); } } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java index 6f01394a9..ab7295a85 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java @@ -22,6 +22,7 @@ import com.iqser.red.service.persistence.management.v1.processor.model.OCRStatus import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileAttributesRepository; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus; @@ -163,6 +164,16 @@ public class FileStatusPersistenceService { } + @Transactional + public void updateStatusErrorInfo(String fileId, FileErrorInfo fileErrorInfo) { + if (fileErrorInfo == null) { + fileRepository.updateStatusErrorInfo(fileId, null, null, null); + } else { + fileRepository.updateStatusErrorInfo(fileId, fileErrorInfo.getCause(), fileErrorInfo.getQueue(), fileErrorInfo.getService()); + } + + } + @Transactional public void updateProcessingStatus(String fileId, ProcessingStatus processingStatus) { @@ -170,6 +181,8 @@ public class FileStatusPersistenceService { return; } if (processingStatus == ProcessingStatus.PROCESSED) { + // reset the error info + fileRepository.updateStatusErrorInfo(fileId, null, null, null); // In case the file is updated to "processed", "lastProcessed" date should be updated to "now" fileRepository.updateProcessingStatus(fileId, processingStatus, diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java index 61ecef7f4..f1e543b50 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java @@ -81,6 +81,10 @@ public interface FileRepository extends JpaRepository { @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated, f.processingErrorCounter = :processingErrorCounter " + "where f.id = :fileId") void updateProcessingStatus(String fileId, ProcessingStatus processingStatus, OffsetDateTime lastUpdated, int processingErrorCounter); + @Modifying + @Query("update FileEntity f set f.errorCause = :cause, f.errorQueue = :queue, f.errorService = :service where f.id = :fileId") + void updateStatusErrorInfo(String fileId, String cause, String queue, String service); + @Modifying(clearAutomatically = true, flushAutomatically = true) @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated, f.lastProcessed = :lastProcessed, f.processingErrorCounter = :processingErrorCounter " + "where f.id = :fileId") diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/FileModelMapper.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/FileModelMapper.java index e01823312..4d071cb41 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/FileModelMapper.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/FileModelMapper.java @@ -3,6 +3,7 @@ package com.iqser.red.service.persistence.management.v1.processor.utils; import java.util.function.BiConsumer; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel; public class FileModelMapper implements BiConsumer { @@ -11,7 +12,7 @@ public class FileModelMapper implements BiConsumer { public void accept(FileEntity fileEntity, FileModel fileModel) { fileEntity.getFileAttributes().forEach(fa -> fileModel.getFileAttributes().put(fa.getFileAttributeId().getFileAttributeConfigId(), fa.getValue())); - + fileModel.setFileErrorInfo(new FileErrorInfo(fileEntity.getErrorCause(), fileEntity.getErrorQueue(), fileEntity.getErrorService())); } } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml index e807ab9d6..1a2cfd855 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml @@ -132,4 +132,6 @@ databaseChangeLog: - include: file: db/changelog/tenant/48-add-watermark-text-alignment.yaml - include: - file: db/changelog/tenant/sql/49-add-keep_overlapping_objects.sql \ No newline at end of file + file: db/changelog/tenant/sql/49-add-keep_overlapping_objects.sql + - include: + file: db/changelog/tenant/50-add-file-status-error-info.yaml diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/50-add-file-status-error-info.yaml b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/50-add-file-status-error-info.yaml new file mode 100644 index 000000000..9ea4cc1c2 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/50-add-file-status-error-info.yaml @@ -0,0 +1,17 @@ +databaseChangeLog: + - changeSet: + id: add-file-status-error-info + author: corinaolariu + changes: + - addColumn: + columns: + - column: + name: error_cause + type: VARCHAR(255) + - column: + name: error_queue + type: VARCHAR(255) + - column: + name: error_service + type: VARCHAR(255) + tableName: file \ No newline at end of file diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileProcessingTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileProcessingTest.java index 669566b3b..905befc26 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileProcessingTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileProcessingTest.java @@ -19,6 +19,7 @@ import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPers import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult; import com.iqser.red.service.persistence.service.v1.api.shared.model.MessageType; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus; @@ -57,7 +58,7 @@ public class FileProcessingTest extends AbstractPersistenceServerServiceTest { assertThat(file.getOcrStartTime()).isNull(); - fileProcessingClient.analysisFailed(dossier.getId(), file.getId()); + fileProcessingClient.analysisFailed(dossier.getId(), file.getId(), new FileErrorInfo()); var loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.ERROR); @@ -68,7 +69,7 @@ public class FileProcessingTest extends AbstractPersistenceServerServiceTest { assertThat(loadedFile.getWorkflowStatus()).isEqualTo(WorkflowStatus.NEW); assertThat(loadedFile.getAnalysisVersion()).isEqualTo(0); - fileProcessingClient.ocrFailed(dossier.getId(), file.getId()); + fileProcessingClient.ocrFailed(dossier.getId(), file.getId(), new FileErrorInfo()); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.ERROR); @@ -81,7 +82,7 @@ public class FileProcessingTest extends AbstractPersistenceServerServiceTest { assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.INDEXING); assertThat(loadedFile.getLastIndexed()).isNull(); - fileProcessingClient.indexingFailed(dossier.getId(), file.getId()); + fileProcessingClient.indexingFailed(dossier.getId(), file.getId(), new FileErrorInfo()); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.ERROR); @@ -102,9 +103,11 @@ public class FileProcessingTest extends AbstractPersistenceServerServiceTest { assertThat(file.getOcrStartTime()).isNull(); - fileProcessingClient.analysisFailed(dossier.getId(), file.getId()); + var queue = "queue1"; + fileProcessingClient.analysisFailed(dossier.getId(), file.getId(), new FileErrorInfo("something failed", queue, "service")); var loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.ERROR); + assertThat(loadedFile.getFileErrorInfo().getQueue()).isEqualTo(queue); fileProcessingClient.analysisSuccessful(dossier.getId(), file.getId(), @@ -129,7 +132,7 @@ public class FileProcessingTest extends AbstractPersistenceServerServiceTest { // assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.DELETED); assertThat(loadedFile.getSoftDeletedTime()).isNotNull(); - fileProcessingClient.indexingFailed(dossier.getId(), file.getId()); + fileProcessingClient.indexingFailed(dossier.getId(), file.getId(), new FileErrorInfo()); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); // assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.DELETED); assertThat(loadedFile.getSoftDeletedTime()).isNotNull(); @@ -202,9 +205,11 @@ public class FileProcessingTest extends AbstractPersistenceServerServiceTest { var file = fileTesterAndProvider.testAndProvideFile(dossier); - fileProcessingClient.analysisFailed(dossier.getId(), file.getId()); + fileProcessingClient.analysisFailed(dossier.getId(), file.getId(), new FileErrorInfo()); var loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.ERROR); + assertThat(loadedFile.getFileErrorInfo()).isNotNull(); + assertThat(loadedFile.getFileErrorInfo().getCause()).isEqualTo("analysisFailed"); reanalysisClient.reanalyzeFilesForDossier(dossier.getId(), List.of(loadedFile.getId()), true); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FileStatus.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FileStatus.java index 3b4af8ea6..894f85630 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FileStatus.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FileStatus.java @@ -3,6 +3,7 @@ package com.iqser.red.service.persistence.service.v1.api.shared.model; import java.time.OffsetDateTime; import java.util.Set; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus; @@ -138,6 +139,8 @@ public class FileStatus { private int analysisVersion; @Schema(description = "Last time the file was indexed in ES.") private OffsetDateTime lastIndexed; + @Schema(description = "The error information for the error state of the file") + private FileErrorInfo fileErrorInfo; @Schema(description = "Shows if this file has been OCRed by us. Last Time of OCR.") public OffsetDateTime getLastOCRTime() { diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/FileErrorInfo.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/FileErrorInfo.java new file mode 100644 index 000000000..0df2b7d95 --- /dev/null +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/FileErrorInfo.java @@ -0,0 +1,17 @@ +package com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@AllArgsConstructor +@NoArgsConstructor +public class FileErrorInfo { + + private String cause; + private String queue; + private String service; +} diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/FileModel.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/FileModel.java index 06158291e..70156257a 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/FileModel.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/FileModel.java @@ -70,6 +70,7 @@ public class FileModel { private OffsetDateTime redactionModificationDate; private OffsetDateTime fileManipulationDate; private boolean hasHighlights; + private FileErrorInfo fileErrorInfo; public boolean isAnalysisRequired() {