Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

RED-3885: Added queues for pdf2image-service, is disabled by default

Browse Source
This commit is contained in:
Philipp Schramm 2022-05-30 16:05:31 +02:00
parent 6ba35f3cfc
commit a7ee9237a0
6 changed files with 119 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/configuration/MessagingConfiguration.java
View File

@ -44,14 +44,15 @@ public class MessagingConfiguration {
public static final String PRE_PROCESSING_QUEUE = "preprocessingQueue";
public static final String PRE_PROCESSING_DLQ = "preprocessingDLQ";
public static final String PDF_2_IMAGE_QUEUE = "pdf2image_request_queue";
public static final String PDF_2_IMAGE_RESPONSE_QUEUE = "pdf2image_response_queue";
public static final String PDF_2_IMAGE_DLQ = "pdf2image_dead_letter_queue";
@Bean
public Queue nerRequestQueue() {
return QueueBuilder.durable(NER_SERVICE_QUEUE)
.withArgument("x-dead-letter-exchange", "")
.withArgument("x-dead-letter-routing-key", NER_SERVICE_DLQ)
.build();
return QueueBuilder.durable(NER_SERVICE_QUEUE).withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", NER_SERVICE_DLQ).build();
}
@ -86,9 +87,7 @@ public class MessagingConfiguration {
public Queue imageResponseQueue() {
return QueueBuilder.durable(IMAGE_SERVICE_RESPONSE_QUEUE)
.withArgument("x-dead-letter-exchange", "")
.withArgument("x-dead-letter-routing-key", IMAGE_SERVICE_DLQ)
.build();
.withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", IMAGE_SERVICE_DLQ).build();
}
@ -99,14 +98,31 @@ public class MessagingConfiguration {
}
@Bean
public Queue pdf2ImageRequestQueue() {
return QueueBuilder.durable(PDF_2_IMAGE_QUEUE).withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", PDF_2_IMAGE_DLQ).build();
}
@Bean
public Queue pdf2ImageResponseQueue() {
return QueueBuilder.durable(PDF_2_IMAGE_RESPONSE_QUEUE).withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", PDF_2_IMAGE_DLQ).build();
}
@Bean
public Queue pdf2ImageResponseDLQ() {
return QueueBuilder.durable(PDF_2_IMAGE_DLQ).build();
}
@Bean
public Queue redactionQueue() {
return QueueBuilder.durable(REDACTION_QUEUE)
.withArgument("x-dead-letter-exchange", "")
.withArgument("x-dead-letter-routing-key", REDACTION_DQL)
.maxPriority(2)
.build();
return QueueBuilder.durable(REDACTION_QUEUE).withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", REDACTION_DQL).maxPriority(2).build();
}

27
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/model/Pdf2ImageServiceRequest.java Normal file
View File

@ -0,0 +1,27 @@
package com.iqser.red.service.peristence.v1.server.model;
import java.util.HashSet;
import java.util.Set;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class Pdf2ImageServiceRequest {
public static final String TARGET_FILE_EXTENSION = "ORIGIN.pdf.gz";
public static final String OPERATION_CONVERSION = "conversion";
private String dossierId;
private String fileId;
private String targetFileExtension;
@Builder.Default
private Set<Integer> pages = new HashSet<>();
private String operation;
}

33
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileService.java
View File

@ -1,32 +1,34 @@
package com.iqser.red.service.peristence.v1.server.service;
import com.google.common.hash.HashFunction;
import com.google.common.hash.Hashing;
import com.iqser.red.service.pdftron.redaction.v1.api.model.ProcessUntouchedDocumentRequest;
import com.iqser.red.service.pdftron.redaction.v1.api.model.UntouchedDocumentResponse;
import java.time.OffsetDateTime;
import java.util.Arrays;
import org.springframework.stereotype.Service;
import com.iqser.red.service.peristence.v1.server.settings.FileManagementServiceSettings;
import com.iqser.red.service.persistence.management.v1.processor.client.PDFTronRedactionClient;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.ApplicationConfigurationEntity;
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
import com.iqser.red.service.persistence.management.v1.processor.service.ApplicationConfigService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ViewedPagesPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.*;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.CommentPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ForceRedactionPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ImageRecategorizationPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.LegalBasisChangePersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.RemoveRedactionPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ResizeRedactionPersistenceService;
import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.AddFileRequest;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
import com.iqser.red.service.search.v1.model.IndexMessageType;
import feign.FeignException;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import java.nio.charset.StandardCharsets;
import java.time.OffsetDateTime;
import java.util.Arrays;
/**
* Provides the internal interface between upload request and the actual persistence.
@ -50,6 +52,7 @@ public class FileService {
private final ResizeRedactionPersistenceService resizeRedactionPersistenceService;
private final IndexingService indexingService;
private final ApplicationConfigService applicationConfigService;
private final FileManagementServiceSettings fileManagementServiceSettings;
public JSONPrimitive<String> upload(AddFileRequest request, boolean keepManualRedactions) {
@ -71,7 +74,6 @@ public class FileService {
ApplicationConfigurationEntity applicationConfigurationEntity = applicationConfigService.getApplicationConfig();
if (existingStatus != null) {
// the file is already uploaded, just reanalyse it.
@ -81,6 +83,11 @@ public class FileService {
log.info("File {} has no status yet, creating one and setting to unprocessed.", request.getFilename());
fileStatusService.createStatus(request.getDossierId(), request.getFileId(), request.getUploader(), request.getFilename(), applicationConfigurationEntity.isRemoveDigitalSignaturesOnUpload());
}
if (fileManagementServiceSettings.isPdf2ImageServiceEnabled()) {
fileStatusService.addToPdf2ImageQueue(request.getDossierId(), request.getFileId());
}
return new JSONPrimitive<>(request.getFileId());
}

56
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java
View File

@ -1,5 +1,19 @@
package com.iqser.red.service.peristence.v1.server.service;
import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import javax.transaction.Transactional;
import org.apache.commons.lang3.StringUtils;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.stereotype.Service;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.Sets;
@ -7,6 +21,7 @@ import com.iqser.red.service.pdftron.redaction.v1.api.model.DocumentRequest;
import com.iqser.red.service.pdftron.redaction.v1.api.model.ProcessUntouchedDocumentRequest;
import com.iqser.red.service.peristence.v1.server.configuration.MessagingConfiguration;
import com.iqser.red.service.peristence.v1.server.model.NerServiceRequest;
import com.iqser.red.service.peristence.v1.server.model.Pdf2ImageServiceRequest;
import com.iqser.red.service.peristence.v1.server.model.image.ImageServiceRequest;
import com.iqser.red.service.peristence.v1.server.settings.FileManagementServiceSettings;
import com.iqser.red.service.peristence.v1.server.utils.FileModelMapper;
@ -19,7 +34,13 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileAttributeConfigPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ViewedPagesPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.*;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.CommentPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ForceRedactionPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ImageRecategorizationPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.LegalBasisChangePersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.RemoveRedactionPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ResizeRedactionPersistenceService;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus;
@ -32,21 +53,6 @@ import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.stereotype.Service;
import javax.transaction.Transactional;
import java.time.OffsetDateTime;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert;
@Slf4j
@Service
@RequiredArgsConstructor
@ -475,6 +481,24 @@ public class FileStatusService {
}
public void addToPdf2ImageQueue(String dossierId, String fileId) {
try {
rabbitTemplate.convertAndSend(MessagingConfiguration.PDF_2_IMAGE_QUEUE, objectMapper.writeValueAsString(Pdf2ImageServiceRequest.builder()
.dossierId(dossierId)
.fileId(fileId)
.targetFileExtension(Pdf2ImageServiceRequest.TARGET_FILE_EXTENSION)
.operation(Pdf2ImageServiceRequest.OPERATION_CONVERSION)
.build()), message -> {
message.getMessageProperties().setPriority(1);
return message;
});
} catch (JsonProcessingException e) {
throw new RuntimeException(e);
}
}
private void addToOcrQueue(String dossierId, String fileId, int priority) {
try {

3
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/settings/FileManagementServiceSettings.java
View File

@ -27,4 +27,7 @@ public class FileManagementServiceSettings {
private boolean storeImageFile = true;
private boolean migrateOnly;
private boolean pdf2ImageServiceEnabled;
}

2
persistence-service-v1/pom.xml
View File

@ -28,7 +28,7 @@
<redaction-service.version>3.114.0</redaction-service.version>
<search-service.version>2.36.0</search-service.version>
<pdftron-redaction-service.version>3.83.0</pdftron-redaction-service.version>
<redaction-report-service.version>3.45.0</redaction-report-service.version>
<redaction-report-service.version>3.47.0</redaction-report-service.version>
</properties>
<dependencyManagement>