diff --git a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/FileModel.java b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/FileModel.java index 2bc7d2fd2..80effc5ae 100644 --- a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/FileModel.java +++ b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/FileModel.java @@ -51,7 +51,10 @@ public class FileModel { private OffsetDateTime approvalDate; private OffsetDateTime lastUploaded; private long analysisDuration; - private OffsetDateTime lastOCRTime; + private OffsetDateTime ocrStartTime; + private Integer numberOfPagesToOCR; + private Integer numberOfOCRedPages; + private OffsetDateTime ocrEndTime; private boolean hasAnnotationComments; private boolean excluded; private boolean excludedFromAutomaticAnalysis; diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java index bc7821d09..489d3b33d 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java @@ -1,35 +1,22 @@ package com.iqser.red.service.persistence.management.v1.processor.entity.dossier; +import com.iqser.red.service.persistence.management.v1.processor.utils.JSONIntegerSetConverter; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.hibernate.annotations.Fetch; +import org.hibernate.annotations.FetchMode; + +import javax.persistence.*; import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; -import javax.persistence.CascadeType; -import javax.persistence.Column; -import javax.persistence.Convert; -import javax.persistence.Entity; -import javax.persistence.EnumType; -import javax.persistence.Enumerated; -import javax.persistence.FetchType; -import javax.persistence.Id; -import javax.persistence.JoinColumn; -import javax.persistence.OneToMany; -import javax.persistence.Table; - -import org.hibernate.annotations.Fetch; -import org.hibernate.annotations.FetchMode; - -import com.iqser.red.service.persistence.management.v1.processor.utils.JSONIntegerSetConverter; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; - -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; - @Data @Builder @AllArgsConstructor @@ -124,8 +111,17 @@ public class FileEntity { @Column private long analysisDuration; - @Column - private OffsetDateTime lastOCRTime; + @Column(name = "ocr_start_time") + private OffsetDateTime ocrStartTime; + + @Column(name = "number_of_pages_to_ocr") + private Integer numberOfPagesToOCR; + + @Column(name = "number_of_ocred_pages") + private Integer numberOfOCRedPages; + + @Column(name = "ocr_end_time") + private OffsetDateTime ocrEndTime; @Column private boolean hasAnnotationComments; @@ -172,6 +168,10 @@ public class FileEntity { @Column private Integer processingErrorCounter; + public OffsetDateTime getLastOCRTime() { + return this.ocrStartTime; + } + public boolean isSoftDeleted() { return deleted != null; } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/model/OCRStatusUpdateResponse.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/model/OCRStatusUpdateResponse.java new file mode 100644 index 000000000..50e94a415 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/model/OCRStatusUpdateResponse.java @@ -0,0 +1,20 @@ +package com.iqser.red.service.persistence.management.v1.processor.model; + + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@AllArgsConstructor +@Builder +public class OCRStatusUpdateResponse { + + private String fileId; + private int numberOfPagesToOCR; + private int numberOfOCRedPages; + private boolean ocrFinished; + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java index 7cc882425..3b0b0c213 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java @@ -1,5 +1,21 @@ package com.iqser.red.service.persistence.management.v1.processor.service.persistence; +import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeConfigEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; +import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException; +import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; +import com.iqser.red.service.persistence.management.v1.processor.model.OCRStatusUpdateResponse; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileAttributesRepository; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.stereotype.Service; + +import javax.transaction.Transactional; import java.time.OffsetDateTime; import java.time.temporal.ChronoUnit; import java.util.HashSet; @@ -8,24 +24,6 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; -import javax.transaction.Transactional; - -import org.apache.commons.lang3.StringUtils; -import org.springframework.stereotype.Service; - -import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeConfigEntity; -import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeEntity; -import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; -import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException; -import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileAttributesRepository; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; - -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; - @Slf4j @Service @RequiredArgsConstructor @@ -132,12 +130,12 @@ public class FileStatusPersistenceService { @Transactional - public void updateLastOCRTime(String fileId, OffsetDateTime time) { + public void updateOCRStartTime(String fileId, OffsetDateTime time) { if (isFileDeleted(fileId)) { return; } - fileRepository.updateLastOCRTime(fileId, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), time); + fileRepository.updateOCRStartTime(fileId, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), time); fileRepository.updateFileModificationDate(fileId, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); } @@ -445,4 +443,7 @@ public class FileStatusPersistenceService { }).collect(Collectors.toList()); } + public void updateOCRStatus(OCRStatusUpdateResponse response) { + fileRepository.updateOCRStatus(response.getFileId(), response.getNumberOfPagesToOCR(), response.getNumberOfOCRedPages(), response.isOcrFinished() ? OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS) : null); + } } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java index b870343fa..521220cfc 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java @@ -1,19 +1,19 @@ package com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository; -import java.time.OffsetDateTime; -import java.util.List; -import java.util.Set; - -import org.springframework.data.jpa.repository.JpaRepository; -import org.springframework.data.jpa.repository.Modifying; -import org.springframework.data.jpa.repository.Query; - import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.projection.FilePageCountsProjection; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.projection.FileProcessingStatusProjection; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.projection.FileWorkflowStatusProjection; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.jpa.repository.Query; + +import javax.transaction.Transactional; +import java.time.OffsetDateTime; +import java.util.List; +import java.util.Set; public interface FileRepository extends JpaRepository { @@ -89,8 +89,8 @@ public interface FileRepository extends JpaRepository { @Modifying - @Query("update FileEntity f set f.lastUpdated = :lastUpdated, f.lastOCRTime = :lastOCRTime where f.id = :fileId") - void updateLastOCRTime(String fileId, OffsetDateTime lastUpdated, OffsetDateTime lastOCRTime); + @Query("update FileEntity f set f.lastUpdated = :lastUpdated, f.ocrStartTime = :ocrStartTime where f.id = :fileId") + void updateOCRStartTime(String fileId, OffsetDateTime lastUpdated, OffsetDateTime ocrStartTime); @Modifying @@ -144,7 +144,7 @@ public interface FileRepository extends JpaRepository { @Modifying(clearAutomatically = true) @Query("update FileEntity f set f.filename = :filename, f.uploader = :uploader, f.processingStatus = :processingStatus, " + "f.workflowStatus = :workflowStatus, f.lastUploaded = :lastUploaded, f.lastUpdated = :lastUpdated, f.fileManipulationDate = :lastUploaded, " + - "f.lastOCRTime = null, f.excluded = false, f.lastProcessed = null, f.lastReviewer = null, f.lastApprover = null, " + + "f.ocrEndTime = null, f.ocrStartTime = null, f.numberOfPagesToOCR = null, f.numberOfOCRedPages = null, f.excluded = false, f.lastProcessed = null, f.lastReviewer = null, f.lastApprover = null, " + "f.assignee = null, f.approvalDate = null, f.numberOfAnalyses = 0, f.lastManualChangeDate = null, f.redactionModificationDate = null, " + "f.dictionaryVersion = 0, f.dossierDictionaryVersion = 0, f.rulesVersion = 0, f.hasImages = false, " + "f.hasHints = false, f.hasRedactions = false, f.hasSuggestions = false, f.hasUpdates = false, " + @@ -235,6 +235,11 @@ public interface FileRepository extends JpaRepository { "d.softDeletedTime is null and d.hardDeletedTime is null and d.archivedTime is null") int countActiveFiles(String dossierTemplateId); + + @Transactional + @Modifying(clearAutomatically = true) + @Query(value = "update FileEntity f set f.numberOfOCRedPages = :numberOfOCRedPages, f.numberOfPagesToOCR = :numberOfPagesToOCR, f.ocrEndTime = :ocrEndTime where f.id = :fileId") + void updateOCRStatus(String fileId, int numberOfPagesToOCR, int numberOfOCRedPages, OffsetDateTime ocrEndTime); } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/configuration/MessagingConfiguration.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/configuration/MessagingConfiguration.java index 2ee4b3492..9a3d23213 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/configuration/MessagingConfiguration.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/configuration/MessagingConfiguration.java @@ -55,6 +55,9 @@ public class MessagingConfiguration { public static final String CV_ANALYSIS_RESPONSE_QUEUE = "cv_analysis_response_queue"; public static final String CV_ANALYSIS_DLQ = "cv_analysis_dead_letter_queue"; + public static final String OCR_STATUS_UPDATE_RESPONSE_QUEUE = "ocr_status_update_response_queue"; + public static final String OCR_STATUS_UPDATE_RESPONSE_DQL = "ocr_status_update_response_dql"; + @Bean public Queue nerRequestQueue() { @@ -119,6 +122,21 @@ public class MessagingConfiguration { } + @Bean + public Queue ocrStatusUpdateResponseQueue() { + + return QueueBuilder.durable(OCR_STATUS_UPDATE_RESPONSE_QUEUE).withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", OCR_STATUS_UPDATE_RESPONSE_DQL).build(); + } + + + @Bean + public Queue ocrStatusUpdateResponseDQL() { + + return QueueBuilder.durable(OCR_STATUS_UPDATE_RESPONSE_DQL).build(); + } + + + @Bean public Queue redactionQueue() { diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ReanalysisController.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ReanalysisController.java index 5e56a39c9..73d7f8b6a 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ReanalysisController.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ReanalysisController.java @@ -69,7 +69,7 @@ public class ReanalysisController implements ReanalysisResource { var relevantFiles = getAllFilesForDossier(dossierId, true); - relevantFiles.stream().filter(fileStatus -> fileStatus.getLastOCRTime() == null).forEach(fileStatus -> fileStatusService.setStatusOcrQueued(dossierId, fileStatus.getId())); + relevantFiles.stream().filter(fileStatus -> fileStatus.getOcrStartTime() == null).forEach(fileStatus -> fileStatusService.setStatusOcrQueued(dossierId, fileStatus.getId())); } @@ -77,7 +77,7 @@ public class ReanalysisController implements ReanalysisResource { var relevantFiles = getRelevantFiles(dossierId, fileIds); - relevantFiles.stream().filter(fileStatus -> fileStatus.getLastOCRTime() == null).forEach(fileStatus -> fileStatusService.setStatusOcrQueued(dossierId, fileStatus.getId())); + relevantFiles.stream().filter(fileStatus -> fileStatus.getOcrStartTime() == null).forEach(fileStatus -> fileStatusService.setStatusOcrQueued(dossierId, fileStatus.getId())); } @@ -93,7 +93,7 @@ public class ReanalysisController implements ReanalysisResource { if (dossierFile.isSoftOrHardDeleted() || dossierFile.getWorkflowStatus().equals(WorkflowStatus.APPROVED)) { throw new ConflictException("Cannot analyse a deleted/approved file"); } - if (dossierFile.getLastOCRTime() != null) { + if (dossierFile.getOcrStartTime() != null) { throw new ConflictException("File already has been OCR processed"); } ocrFiles(dossierId, Sets.newHashSet(fileId)); diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/internal/AdminInterfaceController.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/internal/AdminInterfaceController.java index da55b5508..15e29f06d 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/internal/AdminInterfaceController.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/internal/AdminInterfaceController.java @@ -61,7 +61,7 @@ public class AdminInterfaceController { @PostMapping("/ocr") public void forceOCR(@RequestParam("dossierId") String dossierId, @RequestParam("fileId") String fileId) { - fileStatusService.updateLastOCRTime(fileId); + fileStatusService.updateOCRStartTime(fileId); fileStatusService.setStatusOcrQueued(dossierId, fileId); } @@ -82,16 +82,16 @@ public class AdminInterfaceController { .filter(f -> !f.isExcluded()) .filter(f -> !f.isExcludedFromAutomaticAnalysis()) .filter(f -> !f.isSoftOrHardDeleted()) - .filter(f -> f.getLastOCRTime() != null) + .filter(f -> f.getOcrStartTime() != null) .filter(f -> f.getWorkflowStatus() != WorkflowStatus.APPROVED).collect(Collectors.toList()); for (var file : filesThatRequireOCR) { log.info("Will OCR file: {} from dossier {} with status {} and processing status {} with last OCR time {}", - file.getId(), file.getDossierId(), file.getWorkflowStatus(), file.getProcessingStatus(), file.getLastOCRTime()); + file.getId(), file.getDossierId(), file.getWorkflowStatus(), file.getProcessingStatus(), file.getOcrStartTime()); if (!dryRun) { - fileStatusService.updateLastOCRTime(file.getId()); + fileStatusService.updateOCRStartTime(file.getId()); fileStatusService.setStatusOcrQueued(file.getDossierId(), file.getId()); } @@ -123,7 +123,7 @@ public class AdminInterfaceController { for (var file : filesThatRequireTextReset) { log.info("Will OCR file: {} from dossier {} with status {} and processing status {} with last OCR time {}", - file.getId(), file.getDossierId(), file.getWorkflowStatus(), file.getProcessingStatus(), file.getLastOCRTime()); + file.getId(), file.getDossierId(), file.getWorkflowStatus(), file.getProcessingStatus(), file.getOcrStartTime()); if (!dryRun) { diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusProcessingUpdateService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusProcessingUpdateService.java index d6f1c6ea4..a3376d80a 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusProcessingUpdateService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusProcessingUpdateService.java @@ -1,5 +1,6 @@ package com.iqser.red.service.peristence.v1.server.service; +import com.iqser.red.service.persistence.management.v1.processor.model.OCRStatusUpdateResponse; import org.springframework.retry.support.RetryTemplate; import org.springframework.web.bind.annotation.RestController; @@ -128,7 +129,7 @@ public class FileStatusProcessingUpdateService { retryTemplate.execute(retryContext -> { log.info("OCR Successful for dossier {} and file {}, Attempt to update status: {}", dossierId, fileId, retryContext.getRetryCount()); - fileStatusService.updateLastOCRTime(fileId); + fileStatusService.updateOCRStatus(OCRStatusUpdateResponse.builder().fileId(fileId).ocrFinished(true).build()); fileStatusService.setStatusFullReprocess(dossierId, fileId, false, true); return null; diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java index bdf66ced6..3a4c22c67 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java @@ -1,19 +1,5 @@ package com.iqser.red.service.peristence.v1.server.service; -import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert; - -import java.time.OffsetDateTime; -import java.util.ArrayList; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; - -import javax.transaction.Transactional; - -import org.apache.commons.lang3.StringUtils; -import org.springframework.amqp.rabbit.core.RabbitTemplate; -import org.springframework.stereotype.Service; - import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Sets; @@ -28,17 +14,12 @@ import com.iqser.red.service.peristence.v1.server.utils.FileModelMapper; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; import com.iqser.red.service.persistence.management.v1.processor.exception.InternalServerErrorException; +import com.iqser.red.service.persistence.management.v1.processor.model.OCRStatusUpdateResponse; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileAttributeConfigPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ViewedPagesPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.CommentPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ForceRedactionPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ImageRecategorizationPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.LegalBasisChangePersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.RemoveRedactionPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ResizeRedactionPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.*; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus; @@ -46,10 +27,21 @@ import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.do import com.iqser.red.service.redaction.v1.model.AnalyzeRequest; import com.iqser.red.service.redaction.v1.model.AnalyzeResult; import com.iqser.red.service.redaction.v1.model.MessageType; - import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.amqp.rabbit.core.RabbitTemplate; +import org.springframework.stereotype.Service; + +import javax.transaction.Transactional; +import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert; @Slf4j @Service @@ -454,10 +446,9 @@ public class FileStatusService { } + public void updateOCRStartTime(String fileId) { - public void updateLastOCRTime(String fileId) { - - fileStatusPersistenceService.updateLastOCRTime(fileId, OffsetDateTime.now()); + fileStatusPersistenceService.updateOCRStartTime(fileId, OffsetDateTime.now()); } @@ -713,4 +704,7 @@ public class FileStatusService { return fileAttributeList; } + public void updateOCRStatus(OCRStatusUpdateResponse response) { + fileStatusPersistenceService.updateOCRStatus(response); + } } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/LicenseReportService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/LicenseReportService.java index 4911f41d2..dfbac0b7a 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/LicenseReportService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/LicenseReportService.java @@ -102,10 +102,10 @@ public class LicenseReportService { reportData.setDeletedDate(fileStatus.getDeleted() == null ? null : fileStatus.getDeleted().toInstant()); reportData.setWorkflowStatus(fileStatus.getWorkflowStatus()); reportData.setNumberOfAnalyzedPages(fileStatus.getNumberOfPages()); - reportData.setNumberOfOcrPages(fileStatus.getLastOCRTime() != null ? fileStatus.getNumberOfPages() : 0); + reportData.setNumberOfOcrPages(fileStatus.getOcrStartTime() != null ? fileStatus.getNumberOfPages() : 0); reportData.setAnalysisCount(fileStatus.getNumberOfAnalyses()); result.totalPagesAnalyzed += fileStatus.getNumberOfPages(); - result.totalOcrPages += fileStatus.getLastOCRTime() != null ? fileStatus.getNumberOfPages() : 0; + result.totalOcrPages += fileStatus.getOcrStartTime() != null ? fileStatus.getNumberOfPages() : 0; result.totalNumberOfAnalyses += fileStatus.getNumberOfAnalyses(); result.data.add(reportData); }); diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/OCRProcessingMessageReceiver.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/OCRProcessingMessageReceiver.java new file mode 100644 index 000000000..4967390ab --- /dev/null +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/OCRProcessingMessageReceiver.java @@ -0,0 +1,43 @@ +package com.iqser.red.service.peristence.v1.server.service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iqser.red.service.peristence.v1.server.configuration.MessagingConfiguration; +import com.iqser.red.service.persistence.management.v1.processor.model.OCRStatusUpdateResponse; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.springframework.amqp.core.Message; +import org.springframework.amqp.rabbit.annotation.RabbitListener; +import org.springframework.stereotype.Service; + +@Slf4j +@Service +@RequiredArgsConstructor +public class OCRProcessingMessageReceiver { + + private final ObjectMapper objectMapper; + private final FileStatusService fileStatusService; + + + @SneakyThrows + @RabbitListener(queues = MessagingConfiguration.OCR_STATUS_UPDATE_RESPONSE_QUEUE) + public void receive(String in) { + + var response = objectMapper.readValue(in, OCRStatusUpdateResponse.class); + + fileStatusService.updateOCRStatus(response); + + log.info("Received message {} in {}", response, MessagingConfiguration.OCR_STATUS_UPDATE_RESPONSE_QUEUE); + } + + + @SneakyThrows + @RabbitListener(queues = MessagingConfiguration.OCR_STATUS_UPDATE_RESPONSE_DQL) + public void handleDLQMessage(Message failedMessage) { + + var response = objectMapper.readValue(failedMessage.getBody(), OCRStatusUpdateResponse.class); + + log.info("Received message {} in {}", response, MessagingConfiguration.OCR_STATUS_UPDATE_RESPONSE_DQL); + } + +} diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml b/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml index e9d83f4e0..2e64391d5 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml +++ b/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml @@ -100,4 +100,6 @@ databaseChangeLog: - include: file: db/changelog/tenant/release-3.3.14/1-bump-rules-version.sql - include: - file: db/changelog/tenant/40-remove-quartz.changelog.yaml \ No newline at end of file + file: db/changelog/tenant/40-remove-quartz.changelog.yaml + - include: + file: db/changelog/tenant/sql/41-update-ocr-file-columns.sql diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/tenant/sql/41-update-ocr-file-columns.sql b/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/tenant/sql/41-update-ocr-file-columns.sql new file mode 100644 index 000000000..f596d87c9 --- /dev/null +++ b/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/tenant/sql/41-update-ocr-file-columns.sql @@ -0,0 +1,5 @@ +ALTER TABLE file RENAME COLUMN lastocrtime TO ocr_start_time; +ALTER TABLE file ADD COLUMN ocr_end_time TIMESTAMP WITHOUT TIME ZONE; +update file set ocr_end_time = ocr_start_time; +ALTER TABLE FILE ADD COLUMN number_of_pages_to_ocr BIGINT; +ALTER TABLE FILE ADD COLUMN number_of_ocred_pages BIGINT; diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileProcessingTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileProcessingTest.java index b0a7a3b28..82d43fb66 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileProcessingTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileProcessingTest.java @@ -44,7 +44,7 @@ public class FileProcessingTest extends AbstractPersistenceServerServiceTest { var file = fileTesterAndProvider.testAndProvideFile(dossier); - assertThat(file.getLastOCRTime()).isNull(); + assertThat(file.getOcrStartTime()).isNull(); fileProcessingClient.analysisFailed(dossier.getId(), file.getId()); var loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); @@ -68,7 +68,7 @@ public class FileProcessingTest extends AbstractPersistenceServerServiceTest { fileProcessingClient.ocrSuccessful(dossier.getId(), file.getId()); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.FULL_PROCESSING); - assertThat(loadedFile.getLastOCRTime()).isNotNull(); + assertThat(loadedFile.getOcrEndTime()).isNotNull(); fileProcessingClient.indexing(dossier.getId(), file.getId()); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); @@ -94,7 +94,7 @@ public class FileProcessingTest extends AbstractPersistenceServerServiceTest { var file = fileTesterAndProvider.testAndProvideFile(dossier); - assertThat(file.getLastOCRTime()).isNull(); + assertThat(file.getOcrStartTime()).isNull(); fileProcessingClient.analysisFailed(dossier.getId(), file.getId()); var loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());