Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

Pull request #521: RED-4897: Remove AMQP queues for pdf2image-service

Browse Source 
Merge in RED/persistence-service from RED-4897-ps1 to master

* commit 'ee61750a0057844501aab39d2f88b4cecac8e7f9':
  RED-4897: Remove AMQP queues for pdf2image-service
This commit is contained in:
Ali Oezyetimoglu 2022-08-18 11:44:52 +02:00
commit 0ad0b20dea
6 changed files with 0 additions and 165 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/configuration/MessagingConfiguration.java
View File

@ -44,10 +44,6 @@ public class MessagingConfiguration {
public static final String PRE_PROCESSING_QUEUE = "preprocessingQueue"; public static final String PRE_PROCESSING_QUEUE = "preprocessingQueue";
public static final String PRE_PROCESSING_DLQ = "preprocessingDLQ"; public static final String PRE_PROCESSING_DLQ = "preprocessingDLQ";
public static final String PDF_2_IMAGE_QUEUE = "pdf2image_request_queue";
public static final String PDF_2_IMAGE_RESPONSE_QUEUE = "pdf2image_response_queue";
public static final String PDF_2_IMAGE_DLQ = "pdf2image_dead_letter_queue";
public static final String PDFTRON_QUEUE = "pdftron_queue"; public static final String PDFTRON_QUEUE = "pdftron_queue";
public static final String PDFTRON_DLQ = "pdftron_dlq"; public static final String PDFTRON_DLQ = "pdftron_dlq";
public static final String PDFTRON_RESULT_QUEUE = "pdftron_result_queue"; public static final String PDFTRON_RESULT_QUEUE = "pdftron_result_queue";
@ -99,27 +95,6 @@ public class MessagingConfiguration {
} }
@Bean
public Queue pdf2ImageRequestQueue() {
return QueueBuilder.durable(PDF_2_IMAGE_QUEUE).withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", PDF_2_IMAGE_DLQ).build();
}
@Bean
public Queue pdf2ImageResponseQueue() {
return QueueBuilder.durable(PDF_2_IMAGE_RESPONSE_QUEUE).withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", PDF_2_IMAGE_DLQ).build();
}
@Bean
public Queue pdf2ImageResponseDLQ() {
return QueueBuilder.durable(PDF_2_IMAGE_DLQ).build();
}
@Bean @Bean
public Queue cvAnalysisRequestQueue() { public Queue cvAnalysisRequestQueue() {

27
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/model/Pdf2ImageServiceRequest.java
View File

@ -1,27 +0,0 @@
package com.iqser.red.service.peristence.v1.server.model;
import java.util.HashSet;
import java.util.Set;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class Pdf2ImageServiceRequest {
public static final String TARGET_FILE_EXTENSION = "ORIGIN.pdf.gz";
public static final String OPERATION_CONVERSION = "conversion";
private String dossierId;
private String fileId;
private String targetFileExtension;
@Builder.Default
private Set<Integer> pages = new HashSet<>();
private String operation;
}

27
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/model/Pdf2ImageServiceResponse.java
View File

@ -1,27 +0,0 @@
package com.iqser.red.service.peristence.v1.server.model;
import java.util.HashSet;
import java.util.Set;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class Pdf2ImageServiceResponse {
public static final String TARGET_FILE_EXTENSION = "ORIGIN.pdf.gz";
public static final String OPERATION_CONVERSION = "conversion";
private String dossierId;
private String fileId;
private String targetFileExtension;
@Builder.Default
private Set<Integer> pages = new HashSet<>();
private String operation;
}

27
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java
View File

@ -22,7 +22,6 @@ import com.iqser.red.service.pdftron.redaction.v1.api.model.ProcessUntouchedDocu
import com.iqser.red.service.peristence.v1.server.configuration.MessagingConfiguration; import com.iqser.red.service.peristence.v1.server.configuration.MessagingConfiguration;
import com.iqser.red.service.peristence.v1.server.model.CvAnalysisServiceRequest; import com.iqser.red.service.peristence.v1.server.model.CvAnalysisServiceRequest;
import com.iqser.red.service.peristence.v1.server.model.NerServiceRequest; import com.iqser.red.service.peristence.v1.server.model.NerServiceRequest;
import com.iqser.red.service.peristence.v1.server.model.Pdf2ImageServiceRequest;
import com.iqser.red.service.peristence.v1.server.model.image.ImageServiceRequest; import com.iqser.red.service.peristence.v1.server.model.image.ImageServiceRequest;
import com.iqser.red.service.peristence.v1.server.settings.FileManagementServiceSettings; import com.iqser.red.service.peristence.v1.server.settings.FileManagementServiceSettings;
import com.iqser.red.service.peristence.v1.server.utils.FileModelMapper; import com.iqser.red.service.peristence.v1.server.utils.FileModelMapper;
@ -134,10 +133,6 @@ public class FileStatusService {
fileStatusPersistenceService.updateProcessingStatusPreprocessed(fileId, hasHighlights, fileSize); fileStatusPersistenceService.updateProcessingStatusPreprocessed(fileId, hasHighlights, fileSize);
addToAnalysisQueue(dossierId, fileId, false, Set.of(), false); addToAnalysisQueue(dossierId, fileId, false, Set.of(), false);
if (fileManagementServiceSettings.isPdf2ImageServiceEnabled()) {
addToPdf2ImageQueue(dossierId, fileId, Set.of());
}
} }
@ -417,28 +412,6 @@ public class FileStatusService {
} }
public void addToPdf2ImageQueue(String dossierId, String fileId, Set<Integer> pages) {
try {
rabbitTemplate.convertAndSend(
MessagingConfiguration.PDF_2_IMAGE_QUEUE,
objectMapper.writeValueAsString(Pdf2ImageServiceRequest.builder()
.dossierId(dossierId)
.fileId(fileId)
.targetFileExtension(Pdf2ImageServiceRequest.TARGET_FILE_EXTENSION)
.pages(pages)
.operation(Pdf2ImageServiceRequest.OPERATION_CONVERSION)
.build()),
message -> {
message.getMessageProperties().setPriority(1);
return message;
});
} catch (JsonProcessingException e) {
throw new RuntimeException(e);
}
}
public void addToCvAnalysisRequestQueue(String dossierId, String fileId, Set<Integer> pages) { public void addToCvAnalysisRequestQueue(String dossierId, String fileId, Set<Integer> pages) {
try { try {

57
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/Pdf2ImageMessageReceiver.java
View File

@ -1,57 +0,0 @@
package com.iqser.red.service.peristence.v1.server.service;
import java.io.IOException;
import java.util.HashMap;
import org.springframework.amqp.core.Message;
import org.springframework.amqp.rabbit.annotation.RabbitListener;
import org.springframework.stereotype.Service;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.peristence.v1.server.configuration.MessagingConfiguration;
import com.iqser.red.service.peristence.v1.server.model.Pdf2ImageServiceResponse;
import com.iqser.red.service.peristence.v1.server.settings.FileManagementServiceSettings;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@RequiredArgsConstructor
public class Pdf2ImageMessageReceiver {
private final ObjectMapper objectMapper;
private final FileStatusService fileStatusService;
private final FileManagementServiceSettings settings;
@SneakyThrows
@RabbitListener(queues = MessagingConfiguration.PDF_2_IMAGE_RESPONSE_QUEUE)
public void receive(byte[] in) {
Pdf2ImageServiceResponse response = objectMapper.readValue(in, Pdf2ImageServiceResponse.class);
log.debug("{}", response);
log.info("Received message {} for dossierId {} and fileId {} and pages {}", MessagingConfiguration.PDF_2_IMAGE_RESPONSE_QUEUE, response.getDossierId(), response.getFileId(), response.getPages());
if (settings.isCvServiceEnabled()) {
fileStatusService.addToCvAnalysisRequestQueue(response.getDossierId(), response.getFileId(), response.getPages());
}
}
@RabbitListener(queues = MessagingConfiguration.PDF_2_IMAGE_DLQ)
public void handleDLQMessage(Message failedMessage) throws IOException {
HashMap<String, Object> pdf2ImageResponse = objectMapper.readValue(failedMessage.getBody(), new TypeReference<>() {
});
String dossierId = (String) pdf2ImageResponse.get("dossierId");
String fileId = (String) pdf2ImageResponse.get("fileId");
log.warn("Received message {} for dossierId {} and fileId {}", MessagingConfiguration.PDF_2_IMAGE_DLQ, dossierId, fileId);
}
}

2
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/settings/FileManagementServiceSettings.java
View File

@ -28,8 +28,6 @@ public class FileManagementServiceSettings {
private boolean migrateOnly; private boolean migrateOnly;
private boolean pdf2ImageServiceEnabled;
private int maxErrorRetries = 1; private int maxErrorRetries = 1;
private boolean cvServiceEnabled; private boolean cvServiceEnabled;