From a07369b7b4b59877802f5deadea87edcbdeb4861 Mon Sep 17 00:00:00 2001 From: deiflaender Date: Wed, 10 Aug 2022 10:42:14 +0200 Subject: [PATCH] RED-4782: Enabled pdftron-redaction-service to set files in pocessing states when it takes them out of the queue --- .../dossier/file/ProcessingStatus.java | 2 +- .../FileStatusProcessingUpdateResource.java | 7 ++++ .../FileStatusPersistenceService.java | 7 ++++ .../FileStatusProcessingUpdateController.java | 12 ++++++ .../controller/ReanalysisController.java | 6 +-- .../internal/AdminInterfaceController.java | 19 ++++----- .../FileStatusProcessingUpdateService.java | 39 +++++++++++++++++++ .../v1/server/service/FileStatusService.java | 25 +++++++++--- .../integration/tests/DossierStatsTest.java | 2 +- .../integration/tests/ReanalysisTest.java | 24 +++++------- 10 files changed, 109 insertions(+), 34 deletions(-) diff --git a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/ProcessingStatus.java b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/ProcessingStatus.java index 0cae6cd17..63b71ed06 100644 --- a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/ProcessingStatus.java +++ b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/ProcessingStatus.java @@ -1,6 +1,6 @@ package com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file; public enum ProcessingStatus { - ANALYSE, ERROR, FULLREPROCESS, IMAGE_ANALYZING, INDEXING, NER_ANALYZING, OCR_PROCESSING, PROCESSED, PROCESSING, REPROCESS, SURROUNDING_TEXT_PROCESSING, UNPROCESSED, FULL_PROCESSING, PRE_PROCESSING, PRE_PROCESSED + ANALYSE, ERROR, FULLREPROCESS, IMAGE_ANALYZING, INDEXING, NER_ANALYZING, OCR_PROCESSING_QUEUED, OCR_PROCESSING, PROCESSED, PROCESSING, REPROCESS, SURROUNDING_TEXT_PROCESSING, UNPROCESSED, FULL_PROCESSING, PRE_PROCESSING_QUEUED, PRE_PROCESSING, PRE_PROCESSED } diff --git a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/FileStatusProcessingUpdateResource.java b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/FileStatusProcessingUpdateResource.java index 9eb35e6a9..cbe0ec966 100644 --- a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/FileStatusProcessingUpdateResource.java +++ b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/FileStatusProcessingUpdateResource.java @@ -45,6 +45,13 @@ public interface FileStatusProcessingUpdateResource { @PostMapping(value = STATUS_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE + "/analysis-failed") void analysisFailed(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId); + @ResponseStatus(value = HttpStatus.OK) + @PostMapping(value = STATUS_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE + "/pre-processing") + void preProcessing(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId); + + @ResponseStatus(value = HttpStatus.OK) + @PostMapping(value = STATUS_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE + "/ocr-processing") + void ocrProcessing(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId); @ResponseStatus(value = HttpStatus.OK) @PostMapping(value = STATUS_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE + "/ocr-failed") diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java index 22767fb9f..bb57d29b8 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java @@ -107,6 +107,13 @@ public class FileStatusPersistenceService { } + @Transactional + public void updateProcessingStatus(String fileId, ProcessingStatus processingStatus, int processingErrorCount) { + fileRepository.updateProcessingStatus(fileId, processingStatus, OffsetDateTime.now() + .truncatedTo(ChronoUnit.MILLIS), processingErrorCount); + } + + @Transactional public void updateProcessingStatus(String fileId, ProcessingStatus processingStatus) { diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/FileStatusProcessingUpdateController.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/FileStatusProcessingUpdateController.java index 839884532..6f907a369 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/FileStatusProcessingUpdateController.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/FileStatusProcessingUpdateController.java @@ -54,6 +54,18 @@ public class FileStatusProcessingUpdateController implements FileStatusProcessin } + public void preProcessing(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId) { + + fileStatusProcessingUpdateService.preProcessing(dossierId, fileId); + } + + + public void ocrProcessing(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId) { + + fileStatusProcessingUpdateService.ocrProcessing(dossierId, fileId); + } + + public void ocrFailed(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId) { fileStatusProcessingUpdateService.ocrFailed(dossierId, fileId); diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ReanalysisController.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ReanalysisController.java index ccb766691..4769ca472 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ReanalysisController.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ReanalysisController.java @@ -71,7 +71,7 @@ public class ReanalysisController implements ReanalysisResource { relevantFiles.stream() .filter(fileStatus -> fileStatus.getLastOCRTime() == null) - .forEach(fileStatus -> fileStatusService.setStatusOcrProcessing(dossierId, fileStatus.getId())); + .forEach(fileStatus -> fileStatusService.setStatusOcrQueued(dossierId, fileStatus.getId())); } @@ -81,7 +81,7 @@ public class ReanalysisController implements ReanalysisResource { relevantFiles.stream() .filter(fileStatus -> fileStatus.getLastOCRTime() == null) - .forEach(fileStatus -> fileStatusService.setStatusOcrProcessing(dossierId, fileStatus.getId())); + .forEach(fileStatus -> fileStatusService.setStatusOcrQueued(dossierId, fileStatus.getId())); } @@ -91,7 +91,7 @@ public class ReanalysisController implements ReanalysisResource { dossierPersistenceService.getAndValidateDossier(dossierId); if (force) { - fileStatusService.setStatusOcrProcessing(dossierId, fileId); + fileStatusService.setStatusOcrQueued(dossierId, fileId); } else { FileModel dossierFile = fileStatusService.getStatus(fileId); if (dossierFile.isSoftOrHardDeleted() || dossierFile.getWorkflowStatus().equals(WorkflowStatus.APPROVED)) { diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/internal/AdminInterfaceController.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/internal/AdminInterfaceController.java index e7dea6f76..8e8a9114e 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/internal/AdminInterfaceController.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/internal/AdminInterfaceController.java @@ -1,5 +1,12 @@ package com.iqser.red.service.peristence.v1.server.internal; +import java.util.stream.Collectors; + +import org.springframework.amqp.core.AmqpAdmin; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; import com.iqser.red.service.peristence.v1.server.service.DossierService; import com.iqser.red.service.peristence.v1.server.service.FileManagementStorageService; @@ -7,15 +14,9 @@ import com.iqser.red.service.peristence.v1.server.service.FileStatusService; import com.iqser.red.service.peristence.v1.server.settings.FileManagementServiceSettings; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; + import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.springframework.amqp.core.AmqpAdmin; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.RestController; - -import java.util.stream.Collectors; @Slf4j @RestController @@ -59,7 +60,7 @@ public class AdminInterfaceController { public void forceOCR(@RequestParam("dossierId") String dossierId, @RequestParam("fileId") String fileId) { fileStatusService.updateLastOCRTime(fileId); - fileStatusService.setStatusOcrProcessing(dossierId, fileId); + fileStatusService.setStatusOcrQueued(dossierId, fileId); } @@ -89,7 +90,7 @@ public class AdminInterfaceController { if (!dryRun) { fileStatusService.updateLastOCRTime(file.getId()); - fileStatusService.setStatusOcrProcessing(file.getDossierId(), file.getId()); + fileStatusService.setStatusOcrQueued(file.getDossierId(), file.getId()); } } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusProcessingUpdateService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusProcessingUpdateService.java index a5e54903f..d6f1c6ea4 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusProcessingUpdateService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusProcessingUpdateService.java @@ -5,8 +5,11 @@ import org.springframework.web.bind.annotation.RestController; import com.iqser.red.service.pdftron.redaction.v1.api.model.UntouchedDocumentResponse; import com.iqser.red.service.peristence.v1.server.settings.FileManagementServiceSettings; +import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus; import com.iqser.red.service.redaction.v1.model.AnalyzeResult; import com.iqser.red.service.search.v1.model.IndexMessageType; @@ -26,6 +29,8 @@ public class FileStatusProcessingUpdateService { private final ManualRedactionService manualRedactionService; private final FileManagementServiceSettings settings; private final FileManagementStorageService fileManagementStorageService; + private final FileStatusPersistenceService fileStatusPersistenceService; + public void analysisSuccessful(String dossierId, String fileId, AnalyzeResult analyzeResult) { @@ -78,12 +83,46 @@ public class FileStatusProcessingUpdateService { } + public void preProcessing(String dossierId, String fileId) { + + var fileEntity = fileStatusPersistenceService.getStatus(fileId); + + retryTemplate.execute(retryContext -> { + log.info("Preprocessing dossier {} and file {}, Attempt to update status: {}", dossierId, fileId, retryContext.getRetryCount()); + fileStatusService.setStatusPreProcessing(fileId, fileEntity.getProcessingStatus().equals(ProcessingStatus.PRE_PROCESSING) ? fileEntity.getProcessingErrorCounter() + 1 : 0); + return null; + }); + + var updatedFileEntity = fileStatusPersistenceService.getStatus(fileId); + if(updatedFileEntity.getProcessingErrorCounter() > settings.getMaxErrorRetries()){ + throw new ConflictException(String.format("Max Processing Retries exhausted for dossier %s and file %s with errorCount: %s", dossierId, fileId, updatedFileEntity.getProcessingErrorCounter())); + } + } + + public void preprocessingFailed(String dossierId, String fileId) { setStatusError(dossierId, fileId, "preprocessingFailed"); } + public void ocrProcessing(String dossierId, String fileId) { + + var fileEntity = fileStatusPersistenceService.getStatus(fileId); + + retryTemplate.execute(retryContext -> { + log.info("Ocr processing dossier {} and file {}, Attempt to update status: {}", dossierId, fileId, retryContext.getRetryCount()); + fileStatusService.setStatusOcrProcessing(fileId, fileEntity.getProcessingStatus().equals(ProcessingStatus.PRE_PROCESSING) ? fileEntity.getProcessingErrorCounter() + 1 : 0); + return null; + }); + + var updatedFileEntity = fileStatusPersistenceService.getStatus(fileId); + if(updatedFileEntity.getProcessingErrorCounter() > settings.getMaxErrorRetries()){ + throw new ConflictException(String.format("Max Ocr Retries exhausted for dossier %s and file %s with errorCount: %s", dossierId, fileId, updatedFileEntity.getProcessingErrorCounter())); + } + } + + public void ocrSuccessful(String dossierId, String fileId) { retryTemplate.execute(retryContext -> { diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java index 2dcb8eb34..b08a816a7 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java @@ -149,9 +149,9 @@ public class FileStatusService { } - public void setStatusPreProcessing(String fileId) { + public void setStatusPreProcessingQueued(String fileId) { - fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.PRE_PROCESSING); + fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.PRE_PROCESSING_QUEUED); } @@ -244,7 +244,7 @@ public class FileStatusService { .fileName(filename) .build(); - setStatusPreProcessing(fileId); + setStatusPreProcessingQueued(fileId); rabbitTemplate.convertAndSend(MessagingConfiguration.PRE_PROCESSING_QUEUE, objectMapper.writeValueAsString(processUntouchedDocumentRequest)); } @@ -256,12 +256,14 @@ public class FileStatusService { setStatusReprocess(dossierId, fileId, priority, Sets.newHashSet(), false, false); } + @Transactional public void setStatusReprocessForManual(String dossierId, String fileId, boolean priority) { setStatusReprocess(dossierId, fileId, priority, Sets.newHashSet(), false, true); } + @Transactional public void setStatusReprocess(String dossierId, String fileId, boolean priority, boolean triggeredManually) { @@ -359,7 +361,7 @@ public class FileStatusService { } - public void setStatusOcrProcessing(String dossierId, String fileId) { + public void setStatusOcrQueued(String dossierId, String fileId) { FileEntity fileStatus = fileStatusPersistenceService.getStatus(fileId); @@ -368,11 +370,23 @@ public class FileStatusService { return; } - fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.OCR_PROCESSING); + fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.OCR_PROCESSING_QUEUED); addToOcrQueue(dossierId, fileId, 2); } + public void setStatusPreProcessing(String fileId, int processingErrorCount) { + + fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.PRE_PROCESSING, processingErrorCount); + } + + + public void setStatusOcrProcessing(String fileId, int processingErrorCount) { + + fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.OCR_PROCESSING, processingErrorCount); + } + + public void addToImageQueue(String dossierId, String fileId) { setStatusImageAnalyzing(fileId); @@ -640,7 +654,6 @@ public class FileStatusService { } - private void addToOcrQueue(String dossierId, String fileId, int priority) { try { diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierStatsTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierStatsTest.java index 68304488f..5066a586c 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierStatsTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierStatsTest.java @@ -115,7 +115,7 @@ public class DossierStatsTest extends AbstractPersistenceServerServiceTest { assertThat(dossierStats.isHasSuggestionsFilePresent()).isTrue(); assertThat(dossierStats.isHasUpdatesFilePresent()).isTrue(); assertThat(dossierStats.isHasNoFlagsFilePresent()).isFalse(); - assertThat(dossierStats.getFileCountPerProcessingStatus().get(ProcessingStatus.PRE_PROCESSING)).isEqualTo(2); + assertThat(dossierStats.getFileCountPerProcessingStatus().get(ProcessingStatus.PRE_PROCESSING_QUEUED)).isEqualTo(2); assertThat(dossierStats.getLastFileUpdateDate()).isEqualTo(NOW); assertThat(dossierStats.getFileManipulationDate()).isEqualTo(OLDER_2); } diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ReanalysisTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ReanalysisTest.java index 600cb5f6e..d3300d554 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ReanalysisTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ReanalysisTest.java @@ -1,25 +1,21 @@ package com.iqser.red.service.peristence.v1.server.integration.tests; -import com.iqser.red.service.pdftron.redaction.v1.api.model.*; +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.Set; + +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; + import com.iqser.red.service.peristence.v1.server.integration.client.FileClient; import com.iqser.red.service.peristence.v1.server.integration.client.ReanalysisClient; import com.iqser.red.service.peristence.v1.server.integration.service.DossierTesterAndProvider; import com.iqser.red.service.peristence.v1.server.integration.service.FileTesterAndProvider; import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.AddFileRequest; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus; -import feign.FeignException; -import org.junit.Test; -import org.springframework.beans.factory.annotation.Autowired; - -import java.nio.charset.StandardCharsets; -import java.util.Set; - -import static org.assertj.core.api.Assertions.assertThat; - public class ReanalysisTest extends AbstractPersistenceServerServiceTest { @Autowired @@ -46,18 +42,18 @@ public class ReanalysisTest extends AbstractPersistenceServerServiceTest { reanalysisClient.ocrDossier(dossier.getId()); var loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); - assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.OCR_PROCESSING); + assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.OCR_PROCESSING_QUEUED); resetProcessingStatus(file); reanalysisClient.ocrFile(dossier.getId(), file.getId(),true); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); - assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.OCR_PROCESSING); + assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.OCR_PROCESSING_QUEUED); resetProcessingStatus(file); reanalysisClient.ocrFiles(dossier.getId(), Set.of(file.getId())); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); - assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.OCR_PROCESSING); + assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.OCR_PROCESSING_QUEUED); resetProcessingStatus(file); reanalysisClient.reanalyzeFiles(dossier.getId(), Set.of(file.getId()), true);