Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

Merge branch 'RED-8034' into 'master'

Browse Source 
RED-8034: Always load entityLog from persistence-service because excluded...

Closes RED-8034

See merge request redactmanager/persistence-service!275
This commit is contained in:
Dominique Eifländer 2023-12-19 16:48:07 +01:00
commit f2352413e1
7 changed files with 48 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
buildSrc/src/main/kotlin/com.iqser.red.service.java-conventions.gradle.kts
View File

@ -7,8 +7,8 @@ plugins {
}
val redactionServiceVersion by rootProject.extra { "4.199.0" }
val pdftronRedactionServiceVersion by rootProject.extra { "4.38.0" }
val redactionReportServiceVersion by rootProject.extra { "4.36.0" }
val pdftronRedactionServiceVersion by rootProject.extra { "4.48.0" }
val redactionReportServiceVersion by rootProject.extra { "4.47.0" }
val searchServiceVersion by rootProject.extra { "2.71.0" }
repositories {

2
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadPreparationService.java
View File

@ -71,7 +71,7 @@ public class DownloadPreparationService {
downloadStatus.getFiles().forEach(fileEntity -> {
RedactionMessage message = messageBuilder.fileId(fileEntity.getId()).unapprovedFile(fileEntity.getWorkflowStatus() != WorkflowStatus.APPROVED).includeUnprocessed(reportResultMessage.getIncludeUnprocessed()).build();
RedactionMessage message = messageBuilder.fileId(fileEntity.getId()).unapprovedFile(fileEntity.getWorkflowStatus() != WorkflowStatus.APPROVED).build();
log.info("Sending redaction request for downloadId:{} fileId:{} to pdftron-redaction-queue", downloadId, fileEntity.getId());
rabbitTemplate.convertAndSend(MessagingConfiguration.PDFTRON_QUEUE, message);
});

5
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadProcessorService.java
View File

@ -4,12 +4,9 @@ import java.util.Comparator;
import java.util.List;
import java.util.stream.Collectors;
import jakarta.transaction.Transactional;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.stereotype.Service;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
@ -21,6 +18,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
import com.iqser.red.service.persistence.service.v1.api.shared.model.download.DownloadStatusValue;
import com.iqser.red.service.redaction.report.v1.api.model.ReportRequestMessage;
import jakarta.transaction.Transactional;
import lombok.RequiredArgsConstructor;
@Service
@ -52,7 +50,6 @@ public class DownloadProcessorService {
.downloadId(downloadJob.getStorageId())
.dossierId(dossier.getId())
.dossierTemplateId(dossierService.getDossierById(dossier.getId()).getDossierTemplateId())
.includeUnprocessed(downloadJob.getIncludeUnprocessed())
.fileIds(filenameSortedFileIds)
.templateIds(downloadStatus.getReports().stream().map(ReportTemplateEntity::getTemplateId).collect(Collectors.toSet()))
.build(), 1);

13
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/CvAnalysisMessageReceiver.java
View File

@ -14,6 +14,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.FileSta
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo;
import io.micrometer.observation.ObservationRegistry;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
@ -26,12 +27,14 @@ public class CvAnalysisMessageReceiver {
private final ObjectMapper objectMapper;
private final FileStatusService fileStatusService;
private final FileStatusProcessingUpdateService fileStatusProcessingUpdateService;
private final ObservationRegistry observationRegistry;
@SneakyThrows
@RabbitListener(queues = MessagingConfiguration.CV_ANALYSIS_RESPONSE_QUEUE)
public void receive(CvAnalysisServiceResponse response) {
addFileIdToTrace(response.getFileId());
fileStatusService.setStatusAnalyse(response.getDossierId(), response.getFileId(), false);
log.info("Received message in {} for dossierId {} and fileId {}", MessagingConfiguration.CV_ANALYSIS_RESPONSE_QUEUE, response.getDossierId(), response.getFileId());
@ -43,7 +46,7 @@ public class CvAnalysisMessageReceiver {
public void handleDLQMessage(Message failedMessage) {
var response = objectMapper.readValue(failedMessage.getBody(), CvAnalysisServiceResponse.class);
addFileIdToTrace(response.getFileId());
log.warn("Received message from {} for dossierId {} and fileId {}", MessagingConfiguration.IMAGE_SERVICE_DLQ, response.getDossierId(), response.getFileId());
fileStatusProcessingUpdateService.analysisFailed(response.getDossierId(), response.getFileId(), new FileErrorInfo("cv analysis failed", MessagingConfiguration.IMAGE_SERVICE_DLQ, "cv-analysis-service-v1", OffsetDateTime.now().truncatedTo(
@ -51,4 +54,12 @@ public class CvAnalysisMessageReceiver {
}
// This can be removed if tracing is implemented in python services.
private void addFileIdToTrace(String fileId){
if(observationRegistry.getCurrentObservation() != null){
observationRegistry.getCurrentObservation().highCardinalityKeyValue("fileId", fileId);
}
}
}

12
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/ImageMessageReceiver.java
View File

@ -19,6 +19,7 @@ import com.iqser.red.service.persistence.management.v1.processor.settings.FileMa
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import io.micrometer.observation.ObservationRegistry;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
@ -31,6 +32,7 @@ public class ImageMessageReceiver {
private final FileStatusService fileStatusService;
private final ObjectMapper objectMapper;
private final FileStatusProcessingUpdateService fileStatusProcessingUpdateService;
private final ObservationRegistry observationRegistry;
@SneakyThrows
@ -41,6 +43,7 @@ public class ImageMessageReceiver {
String dossierId = imageResponse.path("dossierId").asText();
String fileId = imageResponse.path("fileId").asText();
addFileIdToTrace(fileId);
fileStatusService.setStatusAnalyse(dossierId, fileId, false);
@ -55,10 +58,19 @@ public class ImageMessageReceiver {
});
String dossierId = (String) imageResponse.get("dossierId");
String fileId = (String) imageResponse.get("fileId");
addFileIdToTrace(fileId);
log.warn("Received message from {} for dossierId {} and fileId {}", MessagingConfiguration.IMAGE_SERVICE_DLQ, dossierId, fileId);
fileStatusProcessingUpdateService.analysisFailed(dossierId, fileId, new FileErrorInfo("image analysis failed", MessagingConfiguration.IMAGE_SERVICE_DLQ, "image-service-v3", OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)));
}
// This can be removed if tracing is implemented in python services.
private void addFileIdToTrace(String fileId){
if(observationRegistry.getCurrentObservation() != null){
observationRegistry.getCurrentObservation().highCardinalityKeyValue("fileId", fileId);
}
}
}

12
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/NerMessageReceiver.java
View File

@ -16,6 +16,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.FileSta
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo;
import io.micrometer.observation.ObservationRegistry;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
@ -28,6 +29,7 @@ public class NerMessageReceiver {
private final FileStatusService fileStatusService;
private final ObjectMapper objectMapper;
private final FileStatusProcessingUpdateService fileStatusProcessingUpdateService;
private final ObservationRegistry observationRegistry;
@SneakyThrows
@ -39,6 +41,7 @@ public class NerMessageReceiver {
String dossierId = (String) entityResponse.get("dossierId");
String fileId = (String) entityResponse.get("fileId");
addFileIdToTrace(fileId);
log.info("Received message {} for dossierId {} and fileId {}", MessagingConfiguration.NER_SERVICE_RESPONSE_QUEUE, dossierId, fileId);
fileStatusService.setStatusAnalyse(dossierId, fileId, false);
@ -52,10 +55,19 @@ public class NerMessageReceiver {
});
String dossierId = (String) entityResponse.get("dossierId");
String fileId = (String) entityResponse.get("fileId");
addFileIdToTrace(fileId);
log.warn("Received message {} for dossierId {} and fileId {}", MessagingConfiguration.NER_SERVICE_DLQ, dossierId, fileId);
fileStatusProcessingUpdateService.analysisFailed(dossierId, fileId,
new FileErrorInfo("ner service failed", MessagingConfiguration.NER_SERVICE_DLQ, "entity-recognition-service-v3", OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)));
}
// This can be removed if tracing is implemented in python services.
private void addFileIdToTrace(String fileId){
if(observationRegistry.getCurrentObservation() != null){
observationRegistry.getCurrentObservation().highCardinalityKeyValue("fileId", fileId);
}
}
}

19
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DownloadPreparationTest.java
View File

@ -4,22 +4,19 @@ import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.when;
import java.io.ByteArrayInputStream;
import java.util.*;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import com.iqser.red.service.persistence.management.v1.processor.service.job.DownloadReadyJob;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DownloadStatusPersistenceService;
import com.knecon.fforesight.tenantcommons.TenantProvider;
import com.knecon.fforesight.tenantcommons.model.TenantResponse;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.mock.web.MockMultipartFile;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionResultDetail;
import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionResultMessage;
import com.iqser.red.service.peristence.v1.server.integration.client.DossierClient;
import com.iqser.red.service.peristence.v1.server.integration.client.DownloadClient;
@ -31,6 +28,8 @@ import com.iqser.red.service.peristence.v1.server.integration.service.FileTester
import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest;
import com.iqser.red.service.persistence.management.v1.processor.service.download.DownloadReportMessageReceiver;
import com.iqser.red.service.persistence.management.v1.processor.service.download.RedactionResultMessageReceiver;
import com.iqser.red.service.persistence.management.v1.processor.service.job.DownloadReadyJob;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DownloadStatusPersistenceService;
import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierTemplateModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatus;
@ -45,6 +44,7 @@ import com.iqser.red.service.redaction.report.v1.api.model.ReportResultMessage;
import com.iqser.red.service.redaction.report.v1.api.model.StoredFileInformation;
import com.iqser.red.storage.commons.service.StorageService;
import com.knecon.fforesight.tenantcommons.TenantContext;
import com.knecon.fforesight.tenantcommons.model.TenantResponse;
import lombok.AccessLevel;
import lombok.SneakyThrows;
@ -93,8 +93,6 @@ public class DownloadPreparationTest extends AbstractPersistenceServerServiceTes
DownloadStatusPersistenceService downloadStatusPersistenceService;
@BeforeEach
public void createTestData() {
@ -140,7 +138,6 @@ public class DownloadPreparationTest extends AbstractPersistenceServerServiceTes
ReportResultMessage reportResultMessage = new ReportResultMessage();
reportResultMessage.setUserId(USER_ID);
reportResultMessage.setDownloadId(downloadId);
reportResultMessage.setIncludeUnprocessed(true);
downloadReportMessageReceiver.receive(reportResultMessage);
redactionResultMessageReceiver.receive(RedactionResultMessage.builder()