From f568cf2eec372e6e73163c6f84273d25efe11fb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominique=20Eifl=C3=A4nder?= Date: Tue, 19 Dec 2023 16:34:09 +0100 Subject: [PATCH] RED-8034: Always load entityLog from persistence-service because excluded pages are calculated there --- ...er.red.service.java-conventions.gradle.kts | 4 ++-- .../download/DownloadPreparationService.java | 2 +- .../download/DownloadProcessorService.java | 5 +---- .../queue/CvAnalysisMessageReceiver.java | 13 ++++++++++++- .../service/queue/ImageMessageReceiver.java | 12 ++++++++++++ .../service/queue/NerMessageReceiver.java | 12 ++++++++++++ .../tests/DownloadPreparationTest.java | 19 ++++++++----------- 7 files changed, 48 insertions(+), 19 deletions(-) diff --git a/buildSrc/src/main/kotlin/com.iqser.red.service.java-conventions.gradle.kts b/buildSrc/src/main/kotlin/com.iqser.red.service.java-conventions.gradle.kts index 92f4c3b29..bc7a0d6a0 100644 --- a/buildSrc/src/main/kotlin/com.iqser.red.service.java-conventions.gradle.kts +++ b/buildSrc/src/main/kotlin/com.iqser.red.service.java-conventions.gradle.kts @@ -7,8 +7,8 @@ plugins { } val redactionServiceVersion by rootProject.extra { "4.199.0" } -val pdftronRedactionServiceVersion by rootProject.extra { "4.38.0" } -val redactionReportServiceVersion by rootProject.extra { "4.36.0" } +val pdftronRedactionServiceVersion by rootProject.extra { "4.48.0" } +val redactionReportServiceVersion by rootProject.extra { "4.47.0" } val searchServiceVersion by rootProject.extra { "2.71.0" } repositories { diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadPreparationService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadPreparationService.java index cc08cba8c..65e583478 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadPreparationService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadPreparationService.java @@ -71,7 +71,7 @@ public class DownloadPreparationService { downloadStatus.getFiles().forEach(fileEntity -> { - RedactionMessage message = messageBuilder.fileId(fileEntity.getId()).unapprovedFile(fileEntity.getWorkflowStatus() != WorkflowStatus.APPROVED).includeUnprocessed(reportResultMessage.getIncludeUnprocessed()).build(); + RedactionMessage message = messageBuilder.fileId(fileEntity.getId()).unapprovedFile(fileEntity.getWorkflowStatus() != WorkflowStatus.APPROVED).build(); log.info("Sending redaction request for downloadId:{} fileId:{} to pdftron-redaction-queue", downloadId, fileEntity.getId()); rabbitTemplate.convertAndSend(MessagingConfiguration.PDFTRON_QUEUE, message); }); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadProcessorService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadProcessorService.java index 74d080583..fe26f20b5 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadProcessorService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadProcessorService.java @@ -4,12 +4,9 @@ import java.util.Comparator; import java.util.List; import java.util.stream.Collectors; -import jakarta.transaction.Transactional; - import org.springframework.amqp.rabbit.core.RabbitTemplate; import org.springframework.stereotype.Service; -import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; @@ -21,6 +18,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist import com.iqser.red.service.persistence.service.v1.api.shared.model.download.DownloadStatusValue; import com.iqser.red.service.redaction.report.v1.api.model.ReportRequestMessage; +import jakarta.transaction.Transactional; import lombok.RequiredArgsConstructor; @Service @@ -52,7 +50,6 @@ public class DownloadProcessorService { .downloadId(downloadJob.getStorageId()) .dossierId(dossier.getId()) .dossierTemplateId(dossierService.getDossierById(dossier.getId()).getDossierTemplateId()) - .includeUnprocessed(downloadJob.getIncludeUnprocessed()) .fileIds(filenameSortedFileIds) .templateIds(downloadStatus.getReports().stream().map(ReportTemplateEntity::getTemplateId).collect(Collectors.toSet())) .build(), 1); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/CvAnalysisMessageReceiver.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/CvAnalysisMessageReceiver.java index 2cd6f3f05..a9dac6693 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/CvAnalysisMessageReceiver.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/CvAnalysisMessageReceiver.java @@ -14,6 +14,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.FileSta import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo; +import io.micrometer.observation.ObservationRegistry; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; @@ -26,12 +27,14 @@ public class CvAnalysisMessageReceiver { private final ObjectMapper objectMapper; private final FileStatusService fileStatusService; private final FileStatusProcessingUpdateService fileStatusProcessingUpdateService; + private final ObservationRegistry observationRegistry; @SneakyThrows @RabbitListener(queues = MessagingConfiguration.CV_ANALYSIS_RESPONSE_QUEUE) public void receive(CvAnalysisServiceResponse response) { + addFileIdToTrace(response.getFileId()); fileStatusService.setStatusAnalyse(response.getDossierId(), response.getFileId(), false); log.info("Received message in {} for dossierId {} and fileId {}", MessagingConfiguration.CV_ANALYSIS_RESPONSE_QUEUE, response.getDossierId(), response.getFileId()); @@ -43,7 +46,7 @@ public class CvAnalysisMessageReceiver { public void handleDLQMessage(Message failedMessage) { var response = objectMapper.readValue(failedMessage.getBody(), CvAnalysisServiceResponse.class); - + addFileIdToTrace(response.getFileId()); log.warn("Received message from {} for dossierId {} and fileId {}", MessagingConfiguration.IMAGE_SERVICE_DLQ, response.getDossierId(), response.getFileId()); fileStatusProcessingUpdateService.analysisFailed(response.getDossierId(), response.getFileId(), new FileErrorInfo("cv analysis failed", MessagingConfiguration.IMAGE_SERVICE_DLQ, "cv-analysis-service-v1", OffsetDateTime.now().truncatedTo( @@ -51,4 +54,12 @@ public class CvAnalysisMessageReceiver { } + // This can be removed if tracing is implemented in python services. + private void addFileIdToTrace(String fileId){ + if(observationRegistry.getCurrentObservation() != null){ + observationRegistry.getCurrentObservation().highCardinalityKeyValue("fileId", fileId); + } + } + + } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/ImageMessageReceiver.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/ImageMessageReceiver.java index 00cd9ea22..30767fc32 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/ImageMessageReceiver.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/ImageMessageReceiver.java @@ -19,6 +19,7 @@ import com.iqser.red.service.persistence.management.v1.processor.settings.FileMa import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; +import io.micrometer.observation.ObservationRegistry; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; @@ -31,6 +32,7 @@ public class ImageMessageReceiver { private final FileStatusService fileStatusService; private final ObjectMapper objectMapper; private final FileStatusProcessingUpdateService fileStatusProcessingUpdateService; + private final ObservationRegistry observationRegistry; @SneakyThrows @@ -41,6 +43,7 @@ public class ImageMessageReceiver { String dossierId = imageResponse.path("dossierId").asText(); String fileId = imageResponse.path("fileId").asText(); + addFileIdToTrace(fileId); fileStatusService.setStatusAnalyse(dossierId, fileId, false); @@ -55,10 +58,19 @@ public class ImageMessageReceiver { }); String dossierId = (String) imageResponse.get("dossierId"); String fileId = (String) imageResponse.get("fileId"); + addFileIdToTrace(fileId); log.warn("Received message from {} for dossierId {} and fileId {}", MessagingConfiguration.IMAGE_SERVICE_DLQ, dossierId, fileId); fileStatusProcessingUpdateService.analysisFailed(dossierId, fileId, new FileErrorInfo("image analysis failed", MessagingConfiguration.IMAGE_SERVICE_DLQ, "image-service-v3", OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS))); } + + // This can be removed if tracing is implemented in python services. + private void addFileIdToTrace(String fileId){ + if(observationRegistry.getCurrentObservation() != null){ + observationRegistry.getCurrentObservation().highCardinalityKeyValue("fileId", fileId); + } + } + } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/NerMessageReceiver.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/NerMessageReceiver.java index 06150cde6..62feed4c6 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/NerMessageReceiver.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/NerMessageReceiver.java @@ -16,6 +16,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.FileSta import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo; +import io.micrometer.observation.ObservationRegistry; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; @@ -28,6 +29,7 @@ public class NerMessageReceiver { private final FileStatusService fileStatusService; private final ObjectMapper objectMapper; private final FileStatusProcessingUpdateService fileStatusProcessingUpdateService; + private final ObservationRegistry observationRegistry; @SneakyThrows @@ -39,6 +41,7 @@ public class NerMessageReceiver { String dossierId = (String) entityResponse.get("dossierId"); String fileId = (String) entityResponse.get("fileId"); + addFileIdToTrace(fileId); log.info("Received message {} for dossierId {} and fileId {}", MessagingConfiguration.NER_SERVICE_RESPONSE_QUEUE, dossierId, fileId); fileStatusService.setStatusAnalyse(dossierId, fileId, false); @@ -52,10 +55,19 @@ public class NerMessageReceiver { }); String dossierId = (String) entityResponse.get("dossierId"); String fileId = (String) entityResponse.get("fileId"); + addFileIdToTrace(fileId); log.warn("Received message {} for dossierId {} and fileId {}", MessagingConfiguration.NER_SERVICE_DLQ, dossierId, fileId); fileStatusProcessingUpdateService.analysisFailed(dossierId, fileId, new FileErrorInfo("ner service failed", MessagingConfiguration.NER_SERVICE_DLQ, "entity-recognition-service-v3", OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS))); } + + // This can be removed if tracing is implemented in python services. + private void addFileIdToTrace(String fileId){ + if(observationRegistry.getCurrentObservation() != null){ + observationRegistry.getCurrentObservation().highCardinalityKeyValue("fileId", fileId); + } + } + } diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DownloadPreparationTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DownloadPreparationTest.java index 366d4f3a8..3f30195c5 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DownloadPreparationTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DownloadPreparationTest.java @@ -4,22 +4,19 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.when; import java.io.ByteArrayInputStream; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; -import com.iqser.red.service.persistence.management.v1.processor.service.job.DownloadReadyJob; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DownloadStatusPersistenceService; -import com.knecon.fforesight.tenantcommons.TenantProvider; -import com.knecon.fforesight.tenantcommons.model.TenantResponse; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.mock.web.MockMultipartFile; import com.fasterxml.jackson.databind.ObjectMapper; -import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionResultDetail; import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionResultMessage; import com.iqser.red.service.peristence.v1.server.integration.client.DossierClient; import com.iqser.red.service.peristence.v1.server.integration.client.DownloadClient; @@ -31,6 +28,8 @@ import com.iqser.red.service.peristence.v1.server.integration.service.FileTester import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest; import com.iqser.red.service.persistence.management.v1.processor.service.download.DownloadReportMessageReceiver; import com.iqser.red.service.persistence.management.v1.processor.service.download.RedactionResultMessageReceiver; +import com.iqser.red.service.persistence.management.v1.processor.service.job.DownloadReadyJob; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DownloadStatusPersistenceService; import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierTemplateModel; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatus; @@ -45,6 +44,7 @@ import com.iqser.red.service.redaction.report.v1.api.model.ReportResultMessage; import com.iqser.red.service.redaction.report.v1.api.model.StoredFileInformation; import com.iqser.red.storage.commons.service.StorageService; import com.knecon.fforesight.tenantcommons.TenantContext; +import com.knecon.fforesight.tenantcommons.model.TenantResponse; import lombok.AccessLevel; import lombok.SneakyThrows; @@ -93,8 +93,6 @@ public class DownloadPreparationTest extends AbstractPersistenceServerServiceTes DownloadStatusPersistenceService downloadStatusPersistenceService; - - @BeforeEach public void createTestData() { @@ -140,7 +138,6 @@ public class DownloadPreparationTest extends AbstractPersistenceServerServiceTes ReportResultMessage reportResultMessage = new ReportResultMessage(); reportResultMessage.setUserId(USER_ID); reportResultMessage.setDownloadId(downloadId); - reportResultMessage.setIncludeUnprocessed(true); downloadReportMessageReceiver.receive(reportResultMessage); redactionResultMessageReceiver.receive(RedactionResultMessage.builder()