Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

RED-4782: Enabled pdftron-redaction-service to set files in pocessing states when it takes them out of the queue

Browse Source
This commit is contained in:
deiflaender 2022-08-10 10:42:14 +02:00
parent 0a573431ca
commit a07369b7b4
10 changed files with 109 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/ProcessingStatus.java
View File

@ -1,6 +1,6 @@
package com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file;
public enum ProcessingStatus {
ANALYSE, ERROR, FULLREPROCESS, IMAGE_ANALYZING, INDEXING, NER_ANALYZING, OCR_PROCESSING, PROCESSED, PROCESSING, REPROCESS, SURROUNDING_TEXT_PROCESSING, UNPROCESSED, FULL_PROCESSING, PRE_PROCESSING, PRE_PROCESSED
ANALYSE, ERROR, FULLREPROCESS, IMAGE_ANALYZING, INDEXING, NER_ANALYZING, OCR_PROCESSING_QUEUED, OCR_PROCESSING, PROCESSED, PROCESSING, REPROCESS, SURROUNDING_TEXT_PROCESSING, UNPROCESSED, FULL_PROCESSING, PRE_PROCESSING_QUEUED, PRE_PROCESSING, PRE_PROCESSED
}

7
persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/FileStatusProcessingUpdateResource.java
View File

@ -45,6 +45,13 @@ public interface FileStatusProcessingUpdateResource {
@PostMapping(value = STATUS_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE + "/analysis-failed")
void analysisFailed(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId);
@ResponseStatus(value = HttpStatus.OK)
@PostMapping(value = STATUS_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE + "/pre-processing")
void preProcessing(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId);
@ResponseStatus(value = HttpStatus.OK)
@PostMapping(value = STATUS_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE + "/ocr-processing")
void ocrProcessing(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId);
@ResponseStatus(value = HttpStatus.OK)
@PostMapping(value = STATUS_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE + "/ocr-failed")

7
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java
View File

@ -107,6 +107,13 @@ public class FileStatusPersistenceService {
}
@Transactional
public void updateProcessingStatus(String fileId, ProcessingStatus processingStatus, int processingErrorCount) {
fileRepository.updateProcessingStatus(fileId, processingStatus, OffsetDateTime.now()
.truncatedTo(ChronoUnit.MILLIS), processingErrorCount);
}
@Transactional
public void updateProcessingStatus(String fileId, ProcessingStatus processingStatus) {

12
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/FileStatusProcessingUpdateController.java
View File

@ -54,6 +54,18 @@ public class FileStatusProcessingUpdateController implements FileStatusProcessin
}
public void preProcessing(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId) {
fileStatusProcessingUpdateService.preProcessing(dossierId, fileId);
}
public void ocrProcessing(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId) {
fileStatusProcessingUpdateService.ocrProcessing(dossierId, fileId);
}
public void ocrFailed(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId) {
fileStatusProcessingUpdateService.ocrFailed(dossierId, fileId);

6
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ReanalysisController.java
View File

@ -71,7 +71,7 @@ public class ReanalysisController implements ReanalysisResource {
relevantFiles.stream()
.filter(fileStatus -> fileStatus.getLastOCRTime() == null)
.forEach(fileStatus -> fileStatusService.setStatusOcrProcessing(dossierId, fileStatus.getId()));
.forEach(fileStatus -> fileStatusService.setStatusOcrQueued(dossierId, fileStatus.getId()));
}
@ -81,7 +81,7 @@ public class ReanalysisController implements ReanalysisResource {
relevantFiles.stream()
.filter(fileStatus -> fileStatus.getLastOCRTime() == null)
.forEach(fileStatus -> fileStatusService.setStatusOcrProcessing(dossierId, fileStatus.getId()));
.forEach(fileStatus -> fileStatusService.setStatusOcrQueued(dossierId, fileStatus.getId()));
}
@ -91,7 +91,7 @@ public class ReanalysisController implements ReanalysisResource {
dossierPersistenceService.getAndValidateDossier(dossierId);
if (force) {
fileStatusService.setStatusOcrProcessing(dossierId, fileId);
fileStatusService.setStatusOcrQueued(dossierId, fileId);
} else {
FileModel dossierFile = fileStatusService.getStatus(fileId);
if (dossierFile.isSoftOrHardDeleted() || dossierFile.getWorkflowStatus().equals(WorkflowStatus.APPROVED)) {

19
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/internal/AdminInterfaceController.java
View File

@ -1,5 +1,12 @@
package com.iqser.red.service.peristence.v1.server.internal;
import java.util.stream.Collectors;
import org.springframework.amqp.core.AmqpAdmin;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import com.iqser.red.service.peristence.v1.server.service.DossierService;
import com.iqser.red.service.peristence.v1.server.service.FileManagementStorageService;
@ -7,15 +14,9 @@ import com.iqser.red.service.peristence.v1.server.service.FileStatusService;
import com.iqser.red.service.peristence.v1.server.settings.FileManagementServiceSettings;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.amqp.core.AmqpAdmin;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import java.util.stream.Collectors;
@Slf4j
@RestController
@ -59,7 +60,7 @@ public class AdminInterfaceController {
public void forceOCR(@RequestParam("dossierId") String dossierId, @RequestParam("fileId") String fileId) {
fileStatusService.updateLastOCRTime(fileId);
fileStatusService.setStatusOcrProcessing(dossierId, fileId);
fileStatusService.setStatusOcrQueued(dossierId, fileId);
}
@ -89,7 +90,7 @@ public class AdminInterfaceController {
if (!dryRun) {
fileStatusService.updateLastOCRTime(file.getId());
fileStatusService.setStatusOcrProcessing(file.getDossierId(), file.getId());
fileStatusService.setStatusOcrQueued(file.getDossierId(), file.getId());
}
}

39
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusProcessingUpdateService.java
View File

@ -5,8 +5,11 @@ import org.springframework.web.bind.annotation.RestController;
import com.iqser.red.service.pdftron.redaction.v1.api.model.UntouchedDocumentResponse;
import com.iqser.red.service.peristence.v1.server.settings.FileManagementServiceSettings;
import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus;
import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
import com.iqser.red.service.search.v1.model.IndexMessageType;
@ -26,6 +29,8 @@ public class FileStatusProcessingUpdateService {
private final ManualRedactionService manualRedactionService;
private final FileManagementServiceSettings settings;
private final FileManagementStorageService fileManagementStorageService;
private final FileStatusPersistenceService fileStatusPersistenceService;
public void analysisSuccessful(String dossierId, String fileId, AnalyzeResult analyzeResult) {
@ -78,12 +83,46 @@ public class FileStatusProcessingUpdateService {
}
public void preProcessing(String dossierId, String fileId) {
var fileEntity = fileStatusPersistenceService.getStatus(fileId);
retryTemplate.execute(retryContext -> {
log.info("Preprocessing dossier {} and file {}, Attempt to update status: {}", dossierId, fileId, retryContext.getRetryCount());
fileStatusService.setStatusPreProcessing(fileId, fileEntity.getProcessingStatus().equals(ProcessingStatus.PRE_PROCESSING) ? fileEntity.getProcessingErrorCounter() + 1 : 0);
return null;
});
var updatedFileEntity = fileStatusPersistenceService.getStatus(fileId);
if(updatedFileEntity.getProcessingErrorCounter() > settings.getMaxErrorRetries()){
throw new ConflictException(String.format("Max Processing Retries exhausted for dossier %s and file %s with errorCount: %s", dossierId, fileId, updatedFileEntity.getProcessingErrorCounter()));
}
}
public void preprocessingFailed(String dossierId, String fileId) {
setStatusError(dossierId, fileId, "preprocessingFailed");
}
public void ocrProcessing(String dossierId, String fileId) {
var fileEntity = fileStatusPersistenceService.getStatus(fileId);
retryTemplate.execute(retryContext -> {
log.info("Ocr processing dossier {} and file {}, Attempt to update status: {}", dossierId, fileId, retryContext.getRetryCount());
fileStatusService.setStatusOcrProcessing(fileId, fileEntity.getProcessingStatus().equals(ProcessingStatus.PRE_PROCESSING) ? fileEntity.getProcessingErrorCounter() + 1 : 0);
return null;
});
var updatedFileEntity = fileStatusPersistenceService.getStatus(fileId);
if(updatedFileEntity.getProcessingErrorCounter() > settings.getMaxErrorRetries()){
throw new ConflictException(String.format("Max Ocr Retries exhausted for dossier %s and file %s with errorCount: %s", dossierId, fileId, updatedFileEntity.getProcessingErrorCounter()));
}
}
public void ocrSuccessful(String dossierId, String fileId) {
retryTemplate.execute(retryContext -> {

25
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java
View File

@ -149,9 +149,9 @@ public class FileStatusService {
}
public void setStatusPreProcessing(String fileId) {
public void setStatusPreProcessingQueued(String fileId) {
fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.PRE_PROCESSING);
fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.PRE_PROCESSING_QUEUED);
}
@ -244,7 +244,7 @@ public class FileStatusService {
.fileName(filename)
.build();
setStatusPreProcessing(fileId);
setStatusPreProcessingQueued(fileId);
rabbitTemplate.convertAndSend(MessagingConfiguration.PRE_PROCESSING_QUEUE, objectMapper.writeValueAsString(processUntouchedDocumentRequest));
}
@ -256,12 +256,14 @@ public class FileStatusService {
setStatusReprocess(dossierId, fileId, priority, Sets.newHashSet(), false, false);
}
@Transactional
public void setStatusReprocessForManual(String dossierId, String fileId, boolean priority) {
setStatusReprocess(dossierId, fileId, priority, Sets.newHashSet(), false, true);
}
@Transactional
public void setStatusReprocess(String dossierId, String fileId, boolean priority, boolean triggeredManually) {
@ -359,7 +361,7 @@ public class FileStatusService {
}
public void setStatusOcrProcessing(String dossierId, String fileId) {
public void setStatusOcrQueued(String dossierId, String fileId) {
FileEntity fileStatus = fileStatusPersistenceService.getStatus(fileId);
@ -368,11 +370,23 @@ public class FileStatusService {
return;
}
fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.OCR_PROCESSING);
fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.OCR_PROCESSING_QUEUED);
addToOcrQueue(dossierId, fileId, 2);
}
public void setStatusPreProcessing(String fileId, int processingErrorCount) {
fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.PRE_PROCESSING, processingErrorCount);
}
public void setStatusOcrProcessing(String fileId, int processingErrorCount) {
fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.OCR_PROCESSING, processingErrorCount);
}
public void addToImageQueue(String dossierId, String fileId) {
setStatusImageAnalyzing(fileId);
@ -640,7 +654,6 @@ public class FileStatusService {
}
private void addToOcrQueue(String dossierId, String fileId, int priority) {
try {

2
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierStatsTest.java
View File

@ -115,7 +115,7 @@ public class DossierStatsTest extends AbstractPersistenceServerServiceTest {
assertThat(dossierStats.isHasSuggestionsFilePresent()).isTrue();
assertThat(dossierStats.isHasUpdatesFilePresent()).isTrue();
assertThat(dossierStats.isHasNoFlagsFilePresent()).isFalse();
assertThat(dossierStats.getFileCountPerProcessingStatus().get(ProcessingStatus.PRE_PROCESSING)).isEqualTo(2);
assertThat(dossierStats.getFileCountPerProcessingStatus().get(ProcessingStatus.PRE_PROCESSING_QUEUED)).isEqualTo(2);
assertThat(dossierStats.getLastFileUpdateDate()).isEqualTo(NOW);
assertThat(dossierStats.getFileManipulationDate()).isEqualTo(OLDER_2);
}

24
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ReanalysisTest.java
View File

@ -1,25 +1,21 @@
package com.iqser.red.service.peristence.v1.server.integration.tests;
import com.iqser.red.service.pdftron.redaction.v1.api.model.*;
import static org.assertj.core.api.Assertions.assertThat;
import java.util.Set;
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
import com.iqser.red.service.peristence.v1.server.integration.client.FileClient;
import com.iqser.red.service.peristence.v1.server.integration.client.ReanalysisClient;
import com.iqser.red.service.peristence.v1.server.integration.service.DossierTesterAndProvider;
import com.iqser.red.service.peristence.v1.server.integration.service.FileTesterAndProvider;
import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.AddFileRequest;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus;
import feign.FeignException;
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
import java.nio.charset.StandardCharsets;
import java.util.Set;
import static org.assertj.core.api.Assertions.assertThat;
public class ReanalysisTest extends AbstractPersistenceServerServiceTest {
@Autowired
@ -46,18 +42,18 @@ public class ReanalysisTest extends AbstractPersistenceServerServiceTest {
reanalysisClient.ocrDossier(dossier.getId());
var loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.OCR_PROCESSING);
assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.OCR_PROCESSING_QUEUED);
resetProcessingStatus(file);
reanalysisClient.ocrFile(dossier.getId(), file.getId(),true);
loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.OCR_PROCESSING);
assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.OCR_PROCESSING_QUEUED);
resetProcessingStatus(file);
reanalysisClient.ocrFiles(dossier.getId(), Set.of(file.getId()));
loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.OCR_PROCESSING);
assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.OCR_PROCESSING_QUEUED);
resetProcessingStatus(file);
reanalysisClient.reanalyzeFiles(dossier.getId(), Set.of(file.getId()), true);