From cff06fbb0e3eceb345c685a0fceafdf3942c12c8 Mon Sep 17 00:00:00 2001 From: deiflaender Date: Mon, 30 May 2022 12:49:30 +0200 Subject: [PATCH] RED-4101: Perform preprocessing async via queue --- .../dossier/file/ProcessingStatus.java | 2 +- .../FileStatusProcessingUpdateResource.java | 14 ++- .../FileStatusPersistenceService.java | 31 +++++-- .../repository/FileRepository.java | 15 +++- .../configuration/MessagingConfiguration.java | 21 +++++ .../FileStatusProcessingUpdateController.java | 16 ++++ .../v1/server/service/FileService.java | 14 +-- .../FileStatusProcessingUpdateService.java | 18 ++++ .../v1/server/service/FileStatusService.java | 87 ++++++++++++++++--- .../ReanalysisRequiredStatusService.java | 2 +- .../service/FileTesterAndProvider.java | 1 + .../integration/tests/DossierStatsTest.java | 2 +- 12 files changed, 184 insertions(+), 39 deletions(-) diff --git a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/ProcessingStatus.java b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/ProcessingStatus.java index 268bdc27b..ca0da62ba 100644 --- a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/ProcessingStatus.java +++ b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/ProcessingStatus.java @@ -1,6 +1,6 @@ package com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file; public enum ProcessingStatus { - ANALYSE, ERROR, FULLREPROCESS, IMAGE_ANALYZING, INDEXING, NER_ANALYZING, OCR_PROCESSING, PROCESSED, PROCESSING, REPROCESS, SURROUNDING_TEXT_PROCESSING, UNPROCESSED, FULL_PROCESSING + ANALYSE, ERROR, FULLREPROCESS, IMAGE_ANALYZING, INDEXING, NER_ANALYZING, OCR_PROCESSING, PROCESSED, PROCESSING, REPROCESS, SURROUNDING_TEXT_PROCESSING, UNPROCESSED, FULL_PROCESSING, PRE_PROCESSING, PRE_PROCESSED, PRE_PROCESSING_FAILED } diff --git a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/FileStatusProcessingUpdateResource.java b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/FileStatusProcessingUpdateResource.java index fdef5cc3b..4102fd79f 100644 --- a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/FileStatusProcessingUpdateResource.java +++ b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/FileStatusProcessingUpdateResource.java @@ -1,6 +1,5 @@ package com.iqser.red.service.persistence.service.v1.api.resources; -import com.iqser.red.service.redaction.v1.model.AnalyzeResult; import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; import org.springframework.web.bind.annotation.PathVariable; @@ -8,6 +7,8 @@ import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.ResponseStatus; +import com.iqser.red.service.pdftron.redaction.v1.api.model.UntouchedDocumentResponse; +import com.iqser.red.service.redaction.v1.model.AnalyzeResult; public interface FileStatusProcessingUpdateResource { @@ -19,6 +20,17 @@ public interface FileStatusProcessingUpdateResource { String FILE_ID_PATH_VARIABLE = "/{" + FILE_ID + "}"; + @ResponseStatus(value = HttpStatus.OK) + @PostMapping(value = STATUS_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE + "/preprocessing-successful", consumes = MediaType.APPLICATION_JSON_VALUE) + void preprocessingSuccessful(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId, + @RequestBody UntouchedDocumentResponse untouchedDocumentResponse); + + + @ResponseStatus(value = HttpStatus.OK) + @PostMapping(value = STATUS_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE + "/preprocessing-failed") + void preprocessingFailed(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId); + + @ResponseStatus(value = HttpStatus.OK) @PostMapping(value = STATUS_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE + "/analysis-successful", consumes = MediaType.APPLICATION_JSON_VALUE) void analysisSuccessful(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId, diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java index 46dcb97cc..535d5915f 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java @@ -35,7 +35,7 @@ public class FileStatusPersistenceService { private final DossierPersistenceService dossierService; - public void createStatus(String dossierId, String fileId, String filename, String uploader, boolean hasHighlights) { + public void createStatus(String dossierId, String fileId, String filename, String uploader) { OffsetDateTime now = OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS); FileEntity file = new FileEntity(); @@ -50,11 +50,30 @@ public class FileStatusPersistenceService { file.setLastUploaded(now); file.setLastUpdated(now); file.setFileManipulationDate(now); - file.setHasHighlights(hasHighlights); fileRepository.save(file); } + @Transactional + public void updateProcessingStatusPreprocessed(String fileId, boolean hasHighlights) { + + if (isFileDeleted(fileId)) { + return; + } + fileRepository.updateProcessingStatus(fileId, ProcessingStatus.PRE_PROCESSED, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), hasHighlights); + } + + + @Transactional + public void updateProcessingStatusPreprocessingFailed(String fileId) { + + if (isFileDeleted(fileId)) { + return; + } + fileRepository.updateProcessingStatus(fileId, ProcessingStatus.PRE_PROCESSING_FAILED, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); + } + + @Transactional public void updateProcessingStatus(String fileId, int numberOfPages, long dictionaryVersion, long rulesVersion, long legalBasisVersion, long duration, long dossierDictionaryVersion, @@ -300,20 +319,20 @@ public class FileStatusPersistenceService { @Transactional - public void overwriteFile(String fileId, String uploader, String filename, boolean hasHighlights, boolean keepManualRedactions) { + public void overwriteFile(String fileId, String uploader, String filename, boolean keepManualRedactions) { int countUpdate = 0; if(keepManualRedactions) { - countUpdate = fileRepository.overwriteFileAndKeepManualRedactions(fileId, filename, uploader, ProcessingStatus.FULLREPROCESS, OffsetDateTime.now() - .truncatedTo(ChronoUnit.MILLIS), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), hasHighlights); + countUpdate = fileRepository.overwriteFileAndKeepManualRedactions(fileId, filename, uploader, ProcessingStatus.UNPROCESSED, OffsetDateTime.now() + .truncatedTo(ChronoUnit.MILLIS), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); } else { fileRepository.findById(fileId).ifPresent((file) -> { file.setExcludedPages(new HashSet<>()); }); - countUpdate = fileRepository.overwriteFile(fileId, filename, uploader, ProcessingStatus.FULLREPROCESS, WorkflowStatus.NEW, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), hasHighlights); + countUpdate = fileRepository.overwriteFile(fileId, filename, uploader, ProcessingStatus.UNPROCESSED, WorkflowStatus.NEW, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); } if (countUpdate == 0) { diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java index 56bd7331d..79bb38bc8 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java @@ -56,6 +56,13 @@ public interface FileRepository extends JpaRepository { " where f.id = :fileId") void updateWorkflowStatus(String fileId, WorkflowStatus workflowStatus, OffsetDateTime lastUpdated, OffsetDateTime approvalDate); + + @Modifying(clearAutomatically = true) + @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated, f.hasHighlights = :hasHighlights " + + "where f.id = :fileId") + void updateProcessingStatus(String fileId, ProcessingStatus processingStatus, OffsetDateTime lastUpdated, boolean hasHighlights); + + @Modifying(clearAutomatically = true) @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated " + "where f.id = :fileId") @@ -129,9 +136,9 @@ public interface FileRepository extends JpaRepository { + "f.assignee = null, f.approvalDate = null, f.numberOfAnalyses = 0, f.lastManualChangeDate = null, f.redactionModificationDate = null, " + "f.dictionaryVersion = 0, f.dossierDictionaryVersion = 0, f.rulesVersion = 0, f.hasImages = false, " + "f.hasHints = false, f.hasRedactions = false, f.hasSuggestions = false, f.hasUpdates = false, " - + "f.deleted = null, f.hardDeletedTime = null, f.hasHighlights = :hasHighlights, f.excludedFromAutomaticAnalysis = false " + "where f.id = :fileId") + + "f.deleted = null, f.hardDeletedTime = null, f.hasHighlights = false, f.excludedFromAutomaticAnalysis = false " + "where f.id = :fileId") int overwriteFile(String fileId, String filename, String uploader, ProcessingStatus processingStatus, WorkflowStatus workflowStatus, OffsetDateTime lastUploaded, - OffsetDateTime lastUpdated, boolean hasHighlights); + OffsetDateTime lastUpdated); @Modifying(clearAutomatically = true) @Query("update FileEntity f set f.filename = :filename, f.uploader = :uploader, f.processingStatus = :processingStatus, " @@ -140,9 +147,9 @@ public interface FileRepository extends JpaRepository { + "f.approvalDate = null, f.numberOfAnalyses = 0, f.lastManualChangeDate = null, f.redactionModificationDate = null, " + "f.dictionaryVersion = 0, f.dossierDictionaryVersion = 0, f.rulesVersion = 0, f.hasImages = false, " + "f.hasHints = false, f.hasRedactions = false, f.hasSuggestions = false, f.hasUpdates = false, " - + "f.deleted = null, f.hardDeletedTime = null, f.hasHighlights = :hasHighlights " + "where f.id = :fileId") + + "f.deleted = null, f.hardDeletedTime = null, f.hasHighlights = false " + "where f.id = :fileId") int overwriteFileAndKeepManualRedactions(String fileId, String filename, String uploader, ProcessingStatus processingStatus, OffsetDateTime lastUploaded, - OffsetDateTime lastUpdated, boolean hasHighlights); + OffsetDateTime lastUpdated); @Query("select count(f) from FileEntity f inner join DossierEntity d on d.id = f.dossierId where d.dossierTemplateId = :dossierTemplateId" + " and ((f.deleted is not null and f.hardDeletedTime is null) or " + diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/configuration/MessagingConfiguration.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/configuration/MessagingConfiguration.java index 759ebacef..50a352129 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/configuration/MessagingConfiguration.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/configuration/MessagingConfiguration.java @@ -41,6 +41,9 @@ public class MessagingConfiguration { public static final String NER_SERVICE_RESPONSE_QUEUE = "entity_response_queue"; public static final String NER_SERVICE_DLQ = "entity_dead_letter_queue"; + public static final String PRE_PROCESSING_QUEUE = "preprocessingQueue"; + public static final String PRE_PROCESSING_DLQ = "preprocessingDLQ"; + @Bean public Queue nerRequestQueue() { @@ -218,4 +221,22 @@ public class MessagingConfiguration { return QueueBuilder.durable(DELETE_FROM_INDEX_DLQ).build(); } + @Bean + public Queue preprocessingQueue() { + + return QueueBuilder.durable(PRE_PROCESSING_QUEUE) + .withArgument("x-dead-letter-exchange", "") + .withArgument("x-dead-letter-routing-key", PRE_PROCESSING_DLQ) + .withArgument("x-max-priority", 2) + .maxPriority(2) + .build(); + } + + + @Bean + public Queue preprocessingDeadLetterQueue() { + + return QueueBuilder.durable(PRE_PROCESSING_DLQ).build(); + } + } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/FileStatusProcessingUpdateController.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/FileStatusProcessingUpdateController.java index 1afe8c6e9..839884532 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/FileStatusProcessingUpdateController.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/FileStatusProcessingUpdateController.java @@ -4,6 +4,7 @@ import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RestController; +import com.iqser.red.service.pdftron.redaction.v1.api.model.UntouchedDocumentResponse; import com.iqser.red.service.peristence.v1.server.service.FileStatusProcessingUpdateService; import com.iqser.red.service.persistence.service.v1.api.resources.FileStatusProcessingUpdateResource; import com.iqser.red.service.redaction.v1.model.AnalyzeResult; @@ -19,6 +20,21 @@ public class FileStatusProcessingUpdateController implements FileStatusProcessin private final FileStatusProcessingUpdateService fileStatusProcessingUpdateService; + public void preprocessingSuccessful(@PathVariable(DOSSIER_ID_PARAM) String dossierId, + @PathVariable(FILE_ID) String fileId, + @RequestBody UntouchedDocumentResponse untouchedDocumentResponse) { + + fileStatusProcessingUpdateService.preprocessingSuccessful(dossierId, fileId, untouchedDocumentResponse); + } + + + public void preprocessingFailed(@PathVariable(DOSSIER_ID_PARAM) String dossierId, + @PathVariable(FILE_ID) String fileId) { + + fileStatusProcessingUpdateService.preprocessingFailed(dossierId, fileId); + } + + public void analysisSuccessful(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId, @RequestBody AnalyzeResult analyzeResult) { diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileService.java index a2bc65bc1..80f68b20a 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileService.java @@ -69,27 +69,17 @@ public class FileService { } } - UntouchedDocumentResponse untouchedDocumentResponse; ApplicationConfigurationEntity applicationConfigurationEntity = applicationConfigService.getApplicationConfig(); - try { - untouchedDocumentResponse = pdfTronRedactionClient.processUntouchedDocument(ProcessUntouchedDocumentRequest.builder() - .dossierId(request.getDossierId()) - .fileId(request.getFileId()).fileName(request.getFilename()) - .removeDigitalSignaturesOnUpload(applicationConfigurationEntity.isRemoveDigitalSignaturesOnUpload()).build()); - } catch (FeignException e) { - log.warn("Failed to optimize file: {}", request.getFilename(), e); - throw new BadRequestException(e.getMessage()); - } if (existingStatus != null) { // the file is already uploaded, just reanalyse it. - fileStatusService.overwriteFile(request.getDossierId(), request.getFileId(), request.getUploader(), request.getFilename(), untouchedDocumentResponse.isHasHighlights(), keepManualRedactions); + fileStatusService.overwriteFile(request.getDossierId(), request.getFileId(), request.getUploader(), request.getFilename(), keepManualRedactions, applicationConfigurationEntity.isRemoveDigitalSignaturesOnUpload()); } else { // the file is new, should create a new status for it. log.info("File {} has no status yet, creating one and setting to unprocessed.", request.getFilename()); - fileStatusService.createStatus(request.getDossierId(), request.getFileId(), request.getUploader(), request.getFilename(), untouchedDocumentResponse.isHasHighlights()); + fileStatusService.createStatus(request.getDossierId(), request.getFileId(), request.getUploader(), request.getFilename(), applicationConfigurationEntity.isRemoveDigitalSignaturesOnUpload()); } return new JSONPrimitive<>(request.getFileId()); } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusProcessingUpdateService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusProcessingUpdateService.java index 2259f9299..8580d4dad 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusProcessingUpdateService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusProcessingUpdateService.java @@ -1,5 +1,6 @@ package com.iqser.red.service.peristence.v1.server.service; +import com.iqser.red.service.pdftron.redaction.v1.api.model.UntouchedDocumentResponse; import com.iqser.red.service.peristence.v1.server.settings.FileManagementServiceSettings; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType; @@ -8,6 +9,8 @@ import com.iqser.red.service.search.v1.model.IndexMessageType; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.retry.support.RetryTemplate; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RestController; @Slf4j @@ -69,6 +72,21 @@ public class FileStatusProcessingUpdateService { } + public void preprocessingSuccessful(String dossierId, String fileId, + UntouchedDocumentResponse untouchedDocumentResponse){ + + fileStatusService.updateProcessingStatusPreprocessed(dossierId, fileId, untouchedDocumentResponse.isHasHighlights()); + } + + + + public void preprocessingFailed(String dossierId, String fileId){ + + fileStatusService.updateProcessingStatusPreprocessingFailed(dossierId, fileId); + } + + + public void ocrSuccessful(String dossierId, String fileId) { retryTemplate.execute(retryContext -> { diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java index ac0406296..4ea70045d 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java @@ -4,14 +4,17 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Sets; import com.iqser.red.service.pdftron.redaction.v1.api.model.DocumentRequest; +import com.iqser.red.service.pdftron.redaction.v1.api.model.ProcessUntouchedDocumentRequest; import com.iqser.red.service.peristence.v1.server.configuration.MessagingConfiguration; import com.iqser.red.service.peristence.v1.server.model.NerServiceRequest; import com.iqser.red.service.peristence.v1.server.model.image.ImageServiceRequest; import com.iqser.red.service.peristence.v1.server.settings.FileManagementServiceSettings; import com.iqser.red.service.peristence.v1.server.utils.FileModelMapper; +import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.ApplicationConfigurationEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; import com.iqser.red.service.persistence.management.v1.processor.exception.InternalServerErrorException; +import com.iqser.red.service.persistence.management.v1.processor.service.ApplicationConfigService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileAttributeConfigPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService; @@ -24,14 +27,19 @@ import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.do import com.iqser.red.service.redaction.v1.model.AnalyzeRequest; import com.iqser.red.service.redaction.v1.model.AnalyzeResult; import com.iqser.red.service.redaction.v1.model.MessageType; + import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; + import org.apache.commons.lang3.StringUtils; import org.springframework.amqp.rabbit.core.RabbitTemplate; import org.springframework.stereotype.Service; import javax.transaction.Transactional; + import java.time.OffsetDateTime; +import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.List; import java.util.Set; @@ -61,9 +69,12 @@ public class FileStatusService { private final FileManagementServiceSettings settings; private final ReanalysisRequiredStatusService reanalysisRequiredStatusService; private final ViewedPagesPersistenceService viewedPagesPersistenceService; + private final ApplicationConfigService applicationConfigService; + @Transactional public List getAllRelevantStatusesForReanalysisScheduler() { + var fileEntities = fileStatusPersistenceService.getAllRelevantStatusesForReanalysisScheduler(); var convertedList = convert(fileEntities, FileModel.class, new FileModelMapper()); return reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(convertedList) @@ -72,6 +83,7 @@ public class FileStatusService { .collect(Collectors.toList()); } + @Transactional public List getActiveFiles(String dossierId) { @@ -89,7 +101,9 @@ public class FileStatusService { return reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(convertedList); } + public List getSoftDeletedForDossierList(List dossierIds) { + var fileEntities = fileStatusPersistenceService.getSoftDeletedFiles(dossierIds); var convertedList = convert(fileEntities, FileModel.class, new FileModelMapper()); return reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(convertedList); @@ -99,12 +113,31 @@ public class FileStatusService { @Transactional public FileModel getStatus(String fileId) { + var fileEntity = fileStatusPersistenceService.getStatus(fileId); var converted = convert(fileEntity, FileModel.class, new FileModelMapper()); return reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(converted); } + public void updateProcessingStatusPreprocessed(String dossierId, String fileId, boolean hasHighlights) { + + fileStatusPersistenceService.updateProcessingStatusPreprocessed(fileId, hasHighlights); + addToAnalysisQueue(dossierId, fileId, false, Set.of()); + } + + + public void updateProcessingStatusPreprocessingFailed(String dossierId, String fileId) { + + // TODO add better logic than always reprocess. + + fileStatusPersistenceService.updateProcessingStatusPreprocessingFailed(fileId); + ApplicationConfigurationEntity applicationConfigurationEntity = applicationConfigService.getApplicationConfig(); + var fileEntity = fileStatusPersistenceService.getStatus(fileId); + addToPreprocessingQueue(dossierId, fileId, fileEntity.getFilename(), applicationConfigurationEntity.isRemoveDigitalSignaturesOnUpload()); + } + + public void setExcludedPages(String fileId, Set excludedPages) { fileStatusPersistenceService.setExcludedPages(fileId, excludedPages); @@ -117,6 +150,12 @@ public class FileStatusService { } + public void setStatusPreProcessing(String fileId) { + + fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.PRE_PROCESSING); + } + + public void setStatusProcessing(String fileId) { fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.PROCESSING); @@ -177,7 +216,8 @@ public class FileStatusService { @Transactional - public void setStatusFullReprocess(String dossierId, String fileId, boolean priority, boolean requiresStructureAnalysis) { + public void setStatusFullReprocess(String dossierId, String fileId, boolean priority, + boolean requiresStructureAnalysis) { FileEntity fileStatus = fileStatusPersistenceService.getStatus(fileId); @@ -196,8 +236,26 @@ public class FileStatusService { } + @SneakyThrows + public void addToPreprocessingQueue(String dossierId, String fileId, String filename, + boolean removeDigitalSignatures) { + + var processUntouchedDocumentRequest = ProcessUntouchedDocumentRequest.builder() + .dossierId(dossierId) + .fileId(fileId) + .fileName(filename) + .removeDigitalSignaturesOnUpload(removeDigitalSignatures) + .build(); + + setStatusPreProcessing(fileId); + + rabbitTemplate.convertAndSend(MessagingConfiguration.PRE_PROCESSING_QUEUE, objectMapper.writeValueAsString(processUntouchedDocumentRequest)); + } + + @Transactional - protected void addToAnalysisQueue(String dossierId, String fileId, boolean priority, Set sectionsToReanalyse) { + protected void addToAnalysisQueue(String dossierId, String fileId, boolean priority, + Set sectionsToReanalyse) { var dossier = dossierPersistenceService.getAndValidateDossier(dossierId); var fileEntity = fileStatusPersistenceService.getStatus(fileId); @@ -210,7 +268,6 @@ public class FileStatusService { var fileModel = convert(fileEntity, FileModel.class, new FileModelMapper()); reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(fileModel, true); - boolean reanalyse = fileModel.isReanalysisRequired(); if (!reanalyse && settings.isImageServiceEnabled() && !fileManagementStorageService.objectExists(dossierId, fileId, FileType.IMAGE_INFO)) { @@ -270,7 +327,8 @@ public class FileStatusService { return MessageType.ANALYSE; } if (reanalyse) { - if (fileModel.getLastFileAttributeChange() != null && fileModel.getLastProcessed().isBefore(fileModel.getLastFileAttributeChange())) { + if (fileModel.getLastFileAttributeChange() != null && fileModel.getLastProcessed() + .isBefore(fileModel.getLastFileAttributeChange())) { return MessageType.ANALYSE; } return MessageType.REANALYSE; @@ -294,10 +352,11 @@ public class FileStatusService { @Transactional - public void createStatus(String dossierId, String fileId, String uploader, String filename, boolean hasHighlights) { + public void createStatus(String dossierId, String fileId, String uploader, String filename, + boolean removeDigitalSignatures) { - fileStatusPersistenceService.createStatus(dossierId, fileId, filename, uploader, hasHighlights); - addToAnalysisQueue(dossierId, fileId, false, Set.of()); + fileStatusPersistenceService.createStatus(dossierId, fileId, filename, uploader); + addToPreprocessingQueue(dossierId, fileId, filename, removeDigitalSignatures); } @@ -435,7 +494,8 @@ public class FileStatusService { } - public void overwriteFile(String dossierId, String fileId, String uploader, String filename, boolean hasHighlights, boolean keepManualRedactions) { + public void overwriteFile(String dossierId, String fileId, String uploader, String filename, + boolean keepManualRedactions, boolean removeDigitalSignatures) { fileManagementStorageService.deleteObject(dossierId, fileId, FileType.REDACTION_LOG); fileManagementStorageService.deleteObject(dossierId, fileId, FileType.SECTION_GRID); @@ -443,16 +503,16 @@ public class FileStatusService { fileManagementStorageService.deleteObject(dossierId, fileId, FileType.TEXT); fileManagementStorageService.deleteObject(dossierId, fileId, FileType.NER_ENTITIES); - if(keepManualRedactions) { - fileStatusPersistenceService.overwriteFile(fileId, uploader, filename, hasHighlights, true); + if (keepManualRedactions) { + fileStatusPersistenceService.overwriteFile(fileId, uploader, filename, true); } else { - fileStatusPersistenceService.overwriteFile(fileId, uploader, filename, hasHighlights, false); + fileStatusPersistenceService.overwriteFile(fileId, uploader, filename, false); deleteManualRedactions(dossierId, fileId); } viewedPagesPersistenceService.deleteForFile(fileId); - setStatusFullReprocess(dossierId, fileId, false, true); + addToPreprocessingQueue(dossierId, fileId, filename, removeDigitalSignatures); } @@ -513,7 +573,8 @@ public class FileStatusService { var config = fileAttributeConfigs.stream() .filter(fac -> fac.getId().equals(fileAttribute.getFileAttributeId().getFileAttributeConfigId())) - .findFirst().orElseThrow(() -> new InternalServerErrorException("File Attribute config not defined for attribute :" + fileAttribute.getFileAttributeId())); + .findFirst() + .orElseThrow(() -> new InternalServerErrorException("File Attribute config not defined for attribute :" + fileAttribute.getFileAttributeId())); com.iqser.red.service.redaction.v1.model.FileAttribute attribute = com.iqser.red.service.redaction.v1.model.FileAttribute.builder() .id(fileAttribute.getFileAttributeId().getFileAttributeConfigId()) diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ReanalysisRequiredStatusService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ReanalysisRequiredStatusService.java index 62fd42900..deb493593 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ReanalysisRequiredStatusService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ReanalysisRequiredStatusService.java @@ -70,7 +70,7 @@ public class ReanalysisRequiredStatusService { } if (ProcessingStatus.PROCESSED.equals(fileStatus.getProcessingStatus()) - || ProcessingStatus.UNPROCESSED.equals(fileStatus.getProcessingStatus()) + || ProcessingStatus.PRE_PROCESSED.equals(fileStatus.getProcessingStatus()) || ignoreProcessingStates) { switch (fileStatus.getWorkflowStatus()) { diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/FileTesterAndProvider.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/FileTesterAndProvider.java index f0bc71e01..c07f125db 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/FileTesterAndProvider.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/FileTesterAndProvider.java @@ -60,6 +60,7 @@ public class FileTesterAndProvider { List.of(RedactionLogEntry.builder().id("annotationId").type("manual").value("value entry").build()), null, 0, 0, 0, 0))); fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.SECTION_GRID, objectMapper.writeValueAsBytes(new SectionGrid())); + fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.ORIGIN, objectMapper.writeValueAsBytes("bytes of the file")); return file; } diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierStatsTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierStatsTest.java index 254f66b6e..68304488f 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierStatsTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierStatsTest.java @@ -115,7 +115,7 @@ public class DossierStatsTest extends AbstractPersistenceServerServiceTest { assertThat(dossierStats.isHasSuggestionsFilePresent()).isTrue(); assertThat(dossierStats.isHasUpdatesFilePresent()).isTrue(); assertThat(dossierStats.isHasNoFlagsFilePresent()).isFalse(); - assertThat(dossierStats.getFileCountPerProcessingStatus().get(ProcessingStatus.FULL_PROCESSING)).isEqualTo(2); + assertThat(dossierStats.getFileCountPerProcessingStatus().get(ProcessingStatus.PRE_PROCESSING)).isEqualTo(2); assertThat(dossierStats.getLastFileUpdateDate()).isEqualTo(NOW); assertThat(dossierStats.getFileManipulationDate()).isEqualTo(OLDER_2); }