From a7ee9237a0767b91038ab5408523ee8ef9f8750c Mon Sep 17 00:00:00 2001 From: Philipp Schramm Date: Mon, 30 May 2022 16:05:31 +0200 Subject: [PATCH] RED-3885: Added queues for pdf2image-service, is disabled by default --- .../configuration/MessagingConfiguration.java | 40 +++++++++---- .../server/model/Pdf2ImageServiceRequest.java | 27 +++++++++ .../v1/server/service/FileService.java | 33 ++++++----- .../v1/server/service/FileStatusService.java | 56 +++++++++++++------ .../FileManagementServiceSettings.java | 3 + persistence-service-v1/pom.xml | 2 +- 6 files changed, 119 insertions(+), 42 deletions(-) create mode 100644 persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/model/Pdf2ImageServiceRequest.java diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/configuration/MessagingConfiguration.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/configuration/MessagingConfiguration.java index 50a352129..2abc99f61 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/configuration/MessagingConfiguration.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/configuration/MessagingConfiguration.java @@ -44,14 +44,15 @@ public class MessagingConfiguration { public static final String PRE_PROCESSING_QUEUE = "preprocessingQueue"; public static final String PRE_PROCESSING_DLQ = "preprocessingDLQ"; + public static final String PDF_2_IMAGE_QUEUE = "pdf2image_request_queue"; + public static final String PDF_2_IMAGE_RESPONSE_QUEUE = "pdf2image_response_queue"; + public static final String PDF_2_IMAGE_DLQ = "pdf2image_dead_letter_queue"; + @Bean public Queue nerRequestQueue() { - return QueueBuilder.durable(NER_SERVICE_QUEUE) - .withArgument("x-dead-letter-exchange", "") - .withArgument("x-dead-letter-routing-key", NER_SERVICE_DLQ) - .build(); + return QueueBuilder.durable(NER_SERVICE_QUEUE).withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", NER_SERVICE_DLQ).build(); } @@ -86,9 +87,7 @@ public class MessagingConfiguration { public Queue imageResponseQueue() { return QueueBuilder.durable(IMAGE_SERVICE_RESPONSE_QUEUE) - .withArgument("x-dead-letter-exchange", "") - .withArgument("x-dead-letter-routing-key", IMAGE_SERVICE_DLQ) - .build(); + .withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", IMAGE_SERVICE_DLQ).build(); } @@ -99,14 +98,31 @@ public class MessagingConfiguration { } + @Bean + public Queue pdf2ImageRequestQueue() { + + return QueueBuilder.durable(PDF_2_IMAGE_QUEUE).withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", PDF_2_IMAGE_DLQ).build(); + } + + + @Bean + public Queue pdf2ImageResponseQueue() { + + return QueueBuilder.durable(PDF_2_IMAGE_RESPONSE_QUEUE).withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", PDF_2_IMAGE_DLQ).build(); + } + + + @Bean + public Queue pdf2ImageResponseDLQ() { + + return QueueBuilder.durable(PDF_2_IMAGE_DLQ).build(); + } + + @Bean public Queue redactionQueue() { - return QueueBuilder.durable(REDACTION_QUEUE) - .withArgument("x-dead-letter-exchange", "") - .withArgument("x-dead-letter-routing-key", REDACTION_DQL) - .maxPriority(2) - .build(); + return QueueBuilder.durable(REDACTION_QUEUE).withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", REDACTION_DQL).maxPriority(2).build(); } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/model/Pdf2ImageServiceRequest.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/model/Pdf2ImageServiceRequest.java new file mode 100644 index 000000000..5d23a0764 --- /dev/null +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/model/Pdf2ImageServiceRequest.java @@ -0,0 +1,27 @@ +package com.iqser.red.service.peristence.v1.server.model; + +import java.util.HashSet; +import java.util.Set; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@AllArgsConstructor +@NoArgsConstructor +public class Pdf2ImageServiceRequest { + + public static final String TARGET_FILE_EXTENSION = "ORIGIN.pdf.gz"; + public static final String OPERATION_CONVERSION = "conversion"; + + private String dossierId; + private String fileId; + private String targetFileExtension; + @Builder.Default + private Set pages = new HashSet<>(); + private String operation; + +} diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileService.java index 80f68b20a..f9a7cf4e4 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileService.java @@ -1,32 +1,34 @@ package com.iqser.red.service.peristence.v1.server.service; -import com.google.common.hash.HashFunction; -import com.google.common.hash.Hashing; -import com.iqser.red.service.pdftron.redaction.v1.api.model.ProcessUntouchedDocumentRequest; -import com.iqser.red.service.pdftron.redaction.v1.api.model.UntouchedDocumentResponse; +import java.time.OffsetDateTime; +import java.util.Arrays; + +import org.springframework.stereotype.Service; + +import com.iqser.red.service.peristence.v1.server.settings.FileManagementServiceSettings; import com.iqser.red.service.persistence.management.v1.processor.client.PDFTronRedactionClient; import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.ApplicationConfigurationEntity; -import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException; import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException; import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; import com.iqser.red.service.persistence.management.v1.processor.service.ApplicationConfigService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ViewedPagesPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.*; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.CommentPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ForceRedactionPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ImageRecategorizationPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.LegalBasisChangePersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.RemoveRedactionPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ResizeRedactionPersistenceService; import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.AddFileRequest; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; import com.iqser.red.service.search.v1.model.IndexMessageType; -import feign.FeignException; + import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Service; - -import java.nio.charset.StandardCharsets; -import java.time.OffsetDateTime; -import java.util.Arrays; /** * Provides the internal interface between upload request and the actual persistence. @@ -50,6 +52,7 @@ public class FileService { private final ResizeRedactionPersistenceService resizeRedactionPersistenceService; private final IndexingService indexingService; private final ApplicationConfigService applicationConfigService; + private final FileManagementServiceSettings fileManagementServiceSettings; public JSONPrimitive upload(AddFileRequest request, boolean keepManualRedactions) { @@ -71,7 +74,6 @@ public class FileService { ApplicationConfigurationEntity applicationConfigurationEntity = applicationConfigService.getApplicationConfig(); - if (existingStatus != null) { // the file is already uploaded, just reanalyse it. @@ -81,6 +83,11 @@ public class FileService { log.info("File {} has no status yet, creating one and setting to unprocessed.", request.getFilename()); fileStatusService.createStatus(request.getDossierId(), request.getFileId(), request.getUploader(), request.getFilename(), applicationConfigurationEntity.isRemoveDigitalSignaturesOnUpload()); } + + if (fileManagementServiceSettings.isPdf2ImageServiceEnabled()) { + fileStatusService.addToPdf2ImageQueue(request.getDossierId(), request.getFileId()); + } + return new JSONPrimitive<>(request.getFileId()); } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java index 4ea70045d..7f9d59f79 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java @@ -1,5 +1,19 @@ package com.iqser.red.service.peristence.v1.server.service; +import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert; + +import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import javax.transaction.Transactional; + +import org.apache.commons.lang3.StringUtils; +import org.springframework.amqp.rabbit.core.RabbitTemplate; +import org.springframework.stereotype.Service; + import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Sets; @@ -7,6 +21,7 @@ import com.iqser.red.service.pdftron.redaction.v1.api.model.DocumentRequest; import com.iqser.red.service.pdftron.redaction.v1.api.model.ProcessUntouchedDocumentRequest; import com.iqser.red.service.peristence.v1.server.configuration.MessagingConfiguration; import com.iqser.red.service.peristence.v1.server.model.NerServiceRequest; +import com.iqser.red.service.peristence.v1.server.model.Pdf2ImageServiceRequest; import com.iqser.red.service.peristence.v1.server.model.image.ImageServiceRequest; import com.iqser.red.service.peristence.v1.server.settings.FileManagementServiceSettings; import com.iqser.red.service.peristence.v1.server.utils.FileModelMapper; @@ -19,7 +34,13 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileAttributeConfigPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ViewedPagesPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.*; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.CommentPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ForceRedactionPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ImageRecategorizationPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.LegalBasisChangePersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.RemoveRedactionPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ResizeRedactionPersistenceService; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus; @@ -32,21 +53,6 @@ import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.springframework.amqp.rabbit.core.RabbitTemplate; -import org.springframework.stereotype.Service; - -import javax.transaction.Transactional; - -import java.time.OffsetDateTime; -import java.time.temporal.ChronoUnit; -import java.util.ArrayList; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; - -import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert; - @Slf4j @Service @RequiredArgsConstructor @@ -475,6 +481,24 @@ public class FileStatusService { } + public void addToPdf2ImageQueue(String dossierId, String fileId) { + + try { + rabbitTemplate.convertAndSend(MessagingConfiguration.PDF_2_IMAGE_QUEUE, objectMapper.writeValueAsString(Pdf2ImageServiceRequest.builder() + .dossierId(dossierId) + .fileId(fileId) + .targetFileExtension(Pdf2ImageServiceRequest.TARGET_FILE_EXTENSION) + .operation(Pdf2ImageServiceRequest.OPERATION_CONVERSION) + .build()), message -> { + message.getMessageProperties().setPriority(1); + return message; + }); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + private void addToOcrQueue(String dossierId, String fileId, int priority) { try { diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/settings/FileManagementServiceSettings.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/settings/FileManagementServiceSettings.java index 06ce8cc70..fb63bd836 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/settings/FileManagementServiceSettings.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/settings/FileManagementServiceSettings.java @@ -27,4 +27,7 @@ public class FileManagementServiceSettings { private boolean storeImageFile = true; private boolean migrateOnly; + + private boolean pdf2ImageServiceEnabled; + } diff --git a/persistence-service-v1/pom.xml b/persistence-service-v1/pom.xml index 282dcc9f1..a1c60723d 100755 --- a/persistence-service-v1/pom.xml +++ b/persistence-service-v1/pom.xml @@ -28,7 +28,7 @@ 3.114.0 2.36.0 3.83.0 - 3.45.0 + 3.47.0