From 6a5da764b5c39f1a1f7dafc13a4b6d76ac2d2067 Mon Sep 17 00:00:00 2001 From: Viktor Seifert Date: Thu, 23 Jun 2022 17:29:16 +0200 Subject: [PATCH 1/9] RED-1098: Added configuration for the pdftron-redaction queue's --- .../configuration/MessagingConfiguration.java | 46 +++++++++---------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/configuration/MessagingConfiguration.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/configuration/MessagingConfiguration.java index 2abc99f61..bdfc6b53a 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/configuration/MessagingConfiguration.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/configuration/MessagingConfiguration.java @@ -48,6 +48,9 @@ public class MessagingConfiguration { public static final String PDF_2_IMAGE_RESPONSE_QUEUE = "pdf2image_response_queue"; public static final String PDF_2_IMAGE_DLQ = "pdf2image_dead_letter_queue"; + public static final String PDFTRON_QUEUE = "pdftron_queue"; + public static final String PDFTRON_RESULT_QUEUE = "pdftron_result_queue"; + @Bean public Queue nerRequestQueue() { @@ -59,10 +62,7 @@ public class MessagingConfiguration { @Bean public Queue nerResponseQueue() { - return QueueBuilder.durable(NER_SERVICE_RESPONSE_QUEUE) - .withArgument("x-dead-letter-exchange", "") - .withArgument("x-dead-letter-routing-key", NER_SERVICE_DLQ) - .build(); + return QueueBuilder.durable(NER_SERVICE_RESPONSE_QUEUE).withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", NER_SERVICE_DLQ).build(); } @@ -76,18 +76,14 @@ public class MessagingConfiguration { @Bean public Queue imageRequestQueue() { - return QueueBuilder.durable(IMAGE_SERVICE_QUEUE) - .withArgument("x-dead-letter-exchange", "") - .withArgument("x-dead-letter-routing-key", IMAGE_SERVICE_DLQ) - .build(); + return QueueBuilder.durable(IMAGE_SERVICE_QUEUE).withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", IMAGE_SERVICE_DLQ).build(); } @Bean public Queue imageResponseQueue() { - return QueueBuilder.durable(IMAGE_SERVICE_RESPONSE_QUEUE) - .withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", IMAGE_SERVICE_DLQ).build(); + return QueueBuilder.durable(IMAGE_SERVICE_RESPONSE_QUEUE).withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", IMAGE_SERVICE_DLQ).build(); } @@ -152,10 +148,7 @@ public class MessagingConfiguration { @Bean public Queue downloadQueue() { - return QueueBuilder.durable(DOWNLOAD_QUEUE) - .withArgument("x-dead-letter-exchange", "") - .withArgument("x-dead-letter-routing-key", DOWNLOAD_DLQ) - .build(); + return QueueBuilder.durable(DOWNLOAD_QUEUE).withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", DOWNLOAD_DLQ).build(); } @@ -188,10 +181,7 @@ public class MessagingConfiguration { @Bean public Queue reportResultQueue() { - return QueueBuilder.durable(REPORT_RESULT_QUEUE) - .withArgument("x-dead-letter-exchange", "") - .withArgument("x-dead-letter-routing-key", REPORT_RESULT_DLQ) - .build(); + return QueueBuilder.durable(REPORT_RESULT_QUEUE).withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", REPORT_RESULT_DLQ).build(); } @@ -205,11 +195,7 @@ public class MessagingConfiguration { @Bean public Queue indexingQueue() { - return QueueBuilder.durable(INDEXING_QUEUE) - .withArgument("x-dead-letter-exchange", "") - .withArgument("x-dead-letter-routing-key", INDEXING_DQL) - .maxPriority(2) - .build(); + return QueueBuilder.durable(INDEXING_QUEUE).withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", INDEXING_DQL).maxPriority(2).build(); } @@ -237,6 +223,7 @@ public class MessagingConfiguration { return QueueBuilder.durable(DELETE_FROM_INDEX_DLQ).build(); } + @Bean public Queue preprocessingQueue() { @@ -255,4 +242,17 @@ public class MessagingConfiguration { return QueueBuilder.durable(PRE_PROCESSING_DLQ).build(); } + + @Bean + public Queue pdfTronQueue() { + + return QueueBuilder.durable(PDFTRON_QUEUE).build(); + } + + @Bean + public Queue pdfTronResultQueue() { + + return QueueBuilder.durable(PDFTRON_RESULT_QUEUE).build(); + } + } From b8bf13d6930fdacbce360802b637e7ce7124ae88 Mon Sep 17 00:00:00 2001 From: Viktor Seifert Date: Mon, 27 Jun 2022 18:11:03 +0200 Subject: [PATCH 2/9] RED-1098: Implemented writing the requests to the pdftron-service to the respective queue instead of sending them via http --- .../download/DownloadPreparationService.java | 116 +++++++++++++----- 1 file changed, 86 insertions(+), 30 deletions(-) diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/DownloadPreparationService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/DownloadPreparationService.java index fc98e375f..875c8610e 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/DownloadPreparationService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/DownloadPreparationService.java @@ -1,7 +1,20 @@ package com.iqser.red.service.peristence.v1.server.service.download; +import java.util.Map; +import java.util.stream.Collectors; + +import javax.transaction.Transactional; + +import org.springframework.amqp.rabbit.core.RabbitTemplate; +import org.springframework.stereotype.Service; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionMessage; import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionRequest; +import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionType; import com.iqser.red.service.peristence.v1.server.client.RedactionClient; +import com.iqser.red.service.peristence.v1.server.configuration.MessagingConfiguration; import com.iqser.red.service.peristence.v1.server.service.FileManagementStorageService; import com.iqser.red.service.peristence.v1.server.service.RedactionLogService; import com.iqser.red.service.peristence.v1.server.utils.FileSystemBackedArchiver; @@ -11,7 +24,11 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.dossier. import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.ReportTemplateEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.download.DownloadStatusEntity; import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.*; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DownloadStatusPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.NotificationPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ReportTemplatePersistenceService; import com.iqser.red.service.persistence.service.v1.api.model.audit.AddNotificationRequest; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DownloadFileType; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType; @@ -19,35 +36,74 @@ import com.iqser.red.service.persistence.service.v1.api.model.download.DownloadS import com.iqser.red.service.redaction.report.v1.api.model.ReportResultMessage; import com.iqser.red.service.redaction.report.v1.api.model.StoredFileInformation; import com.iqser.red.service.redaction.v1.model.RedactionLog; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Service; -import javax.transaction.Transactional; -import java.util.Map; -import java.util.stream.Collectors; +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; +import lombok.experimental.FieldDefaults; +import lombok.extern.slf4j.Slf4j; @Slf4j @Service @RequiredArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true) public class DownloadPreparationService { - private final DownloadStatusPersistenceService downloadStatusPersistenceService; - private final FileStatusPersistenceService fileStatusPersistenceService; - private final FileManagementStorageService fileManagementStorageService; - private final RedactionClient redactionClient; - private final PDFTronRedactionClient pdfTronRedactionClient; - private final ReportTemplatePersistenceService reportTemplatePersistenceService; - private final RedactionLogService redactionLogService; - private final NotificationPersistenceService notificationPersistenceService; - private final DossierPersistenceService dossierPersistenceService; + DownloadStatusPersistenceService downloadStatusPersistenceService; + FileStatusPersistenceService fileStatusPersistenceService; + FileManagementStorageService fileManagementStorageService; + RedactionClient redactionClient; + PDFTronRedactionClient pdfTronRedactionClient; + ReportTemplatePersistenceService reportTemplatePersistenceService; + RedactionLogService redactionLogService; + NotificationPersistenceService notificationPersistenceService; + DossierPersistenceService dossierPersistenceService; + RabbitTemplate rabbitTemplate; + ObjectMapper objectMapper; + @Transactional - public void createDownload(ReportResultMessage reportResultMessage) { + public void createDownload(ReportResultMessage reportResultMessage) throws JsonProcessingException { DownloadStatusEntity downloadStatus = downloadStatusPersistenceService.getStatus(reportResultMessage.getDownloadId()); DossierEntity dossier = downloadStatus.getDossier(); + RedactionMessage message = RedactionMessage.builder().build(); + + for (String fileId : downloadStatus.getFiles().stream().map(FileEntity::getId).collect(Collectors.toUnmodifiableList())) { + for (DownloadFileType downloadFileType : downloadStatus.getDownloadFileTypes()) { + RedactionType redactionType = toRedactionType(downloadFileType); + if (redactionType == null) { + continue; + } + + message.getRedactionTypes().add(redactionType); + message.getFileIds().add(fileId); + message.getDossierIds().add(dossier.getId()); + } + } + + rabbitTemplate.convertAndSend(MessagingConfiguration.PDFTRON_QUEUE, objectMapper.writeValueAsString(message)); + } + + + private RedactionType toRedactionType(DownloadFileType downloadFileType) { + + switch (downloadFileType) { + case REDACTED: + return RedactionType.REDACTED; + case PREVIEW: + return RedactionType.PREVIEW; + case DELTA_PREVIEW: + return RedactionType.DELTA; + case ORIGINAL: + default: + return null; + } + } + + + private void createDownloadArchive(ReportResultMessage reportResultMessage, DownloadStatusEntity downloadStatus, DossierEntity dossier) { + try (FileSystemBackedArchiver fileSystemBackedArchiver = new FileSystemBackedArchiver()) { generateAndAddFiles(downloadStatus, fileSystemBackedArchiver); @@ -87,20 +143,16 @@ public class DownloadPreparationService { fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel("Original", fileStatus.getFilename(), original)); } if (downloadFileType.name().equals(DownloadFileType.PREVIEW.name())) { - fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel("Preview", addSuffix(fileStatus.getFilename(), "highlighted"), - getPreview(dossier.getId(), fileId, redactionLog, dossier.getDossierTemplateId(), dossier.getPreviewWatermarkId()))); + fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel("Preview", addSuffix(fileStatus.getFilename(), "highlighted"), getPreview(dossier.getId(), fileId, redactionLog, dossier.getDossierTemplateId(), dossier.getPreviewWatermarkId()))); } if (downloadFileType.name().equals(DownloadFileType.DELTA_PREVIEW.name())) { - fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel("Delta Preview", addSuffix(fileStatus.getFilename(), "delta_highlighted"), - getDeltaPreview(dossier.getId(), fileId, redactionLog, dossier.getDossierTemplateId(), dossier.getPreviewWatermarkId()))); + fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel("Delta Preview", addSuffix(fileStatus.getFilename(), "delta_highlighted"), getDeltaPreview(dossier.getId(), fileId, redactionLog, dossier.getDossierTemplateId(), dossier.getPreviewWatermarkId()))); } if (downloadFileType.name().equals(DownloadFileType.REDACTED.name())) { - fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel("Redacted", addSuffix(fileStatus.getFilename(), "redacted"), - getRedacted(dossier.getId(), fileId, redactionLog, dossier.getDossierTemplateId(), dossier.getWatermarkId()))); + fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel("Redacted", addSuffix(fileStatus.getFilename(), "redacted"), getRedacted(dossier.getId(), fileId, redactionLog, dossier.getDossierTemplateId(), dossier.getWatermarkId()))); } } - log.info("Successfully added file {}/{} for downloadId {}, took {}", i, fileIds - .size(), downloadStatus.getStorageId(), System.currentTimeMillis() - start); + log.info("Successfully added file {}/{} for downloadId {}, took {}", i, fileIds.size(), downloadStatus.getStorageId(), System.currentTimeMillis() - start); i++; } log.info("Successfully added {} files for downloadId {}, took {}", fileIds, downloadStatus.getStorageId(), System.currentTimeMillis() - fileGenerationStart); @@ -113,7 +165,6 @@ public class DownloadPreparationService { for (StoredFileInformation storedFileInformation : reportResultMessage.getStoredFileInformation()) { - FileEntity fileStatus = null; if (storedFileInformation.getFileId() != null) { @@ -122,14 +173,14 @@ public class DownloadPreparationService { ReportTemplateEntity reportTemplate = reportTemplatePersistenceService.find(storedFileInformation.getTemplateId()); byte[] report = fileManagementStorageService.getStoredObjectBytes(storedFileInformation.getStorageId()); - fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(removeExtension(reportTemplate.getFileName()) + (reportTemplate.isMultiFileReport() ? " (multifile)" : ""), - addSuffix(createFileName(fileStatus, reportTemplate), "justification"), report)); + fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(removeExtension(reportTemplate.getFileName()) + (reportTemplate.isMultiFileReport() ? " (multifile)" : ""), addSuffix(createFileName(fileStatus, reportTemplate), "justification"), report)); } log.info("Successfully added {} reports for downloadId {}, took {}", reportResultMessage.getStoredFileInformation() .size(), reportResultMessage.getDownloadId(), System.currentTimeMillis() - addReportsStart); } + private String createFileName(FileEntity fileStatus, ReportTemplateEntity reportTemplate) { if (fileStatus != null) { @@ -139,7 +190,9 @@ public class DownloadPreparationService { } } + private String getExtension(String fileName) { + var index = fileName.lastIndexOf("."); if (index > 0) { return fileName.substring(index); @@ -148,7 +201,9 @@ public class DownloadPreparationService { } } + private String removeExtension(String fileName) { + var index = fileName.lastIndexOf("."); if (index > 0) { return fileName.substring(0, index); @@ -159,6 +214,7 @@ public class DownloadPreparationService { private String addSuffix(String filename, String suffix) { + var oldExtension = getExtension(filename); return removeExtension(filename) + "_" + suffix + oldExtension; @@ -172,8 +228,8 @@ public class DownloadPreparationService { log.info("Successfully stored zip for downloadId {}, took {}", downloadStatus.getStorageId(), System.currentTimeMillis() - start); } - private byte[] getRedacted(String dossierId, String fileId, RedactionLog redactionLog, String dossierTemplateId, - Long watermarkId) { + + private byte[] getRedacted(String dossierId, String fileId, RedactionLog redactionLog, String dossierTemplateId, Long watermarkId) { return pdfTronRedactionClient.redact(RedactionRequest.builder() .dossierId(dossierId) From 4911fbff9c3f983c4217d568bdcce6ba0833e7f7 Mon Sep 17 00:00:00 2001 From: Viktor Seifert Date: Tue, 28 Jun 2022 17:23:28 +0200 Subject: [PATCH 3/9] RED-1098: Added the information about the generated reports to the db & implemented processing the response from pdf-tron. * Added the information about the generated reports to the download-entity so that it is available when the result from pdf-tron is received. * Implemented processing the response from pdf-tron to complete the download archive, using the pdf-tron results from the queue instead of making rest-calls. --- .../persistence-service-processor-v1/pom.xml | 4 + .../entity/download/DownloadStatusEntity.java | 71 ++++--- .../DownloadStatusPersistenceService.java | 25 ++- .../JSONStoredFileInformationConverter.java | 42 ++++ .../download/DownloadPreparationService.java | 194 +++++++++--------- .../RedactionResultMessageReceiver.java | 33 +++ ...-reports-information-column.changelog.yaml | 11 + .../db/changelog/db.changelog-master.yaml | 2 + 8 files changed, 245 insertions(+), 137 deletions(-) create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/JSONStoredFileInformationConverter.java create mode 100644 persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/RedactionResultMessageReceiver.java create mode 100644 persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/34-add-reports-information-column.changelog.yaml diff --git a/persistence-service-v1/persistence-service-processor-v1/pom.xml b/persistence-service-v1/persistence-service-processor-v1/pom.xml index 907c93920..ed3b3ddb8 100644 --- a/persistence-service-v1/persistence-service-processor-v1/pom.xml +++ b/persistence-service-v1/persistence-service-processor-v1/pom.xml @@ -100,5 +100,9 @@ guava compile + + com.iqser.red.service + redaction-report-service-api-v1 + diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/download/DownloadStatusEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/download/DownloadStatusEntity.java index 460568f75..3b17c81de 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/download/DownloadStatusEntity.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/download/DownloadStatusEntity.java @@ -1,62 +1,81 @@ package com.iqser.red.service.persistence.management.v1.processor.entity.download; -import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity; -import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; -import com.iqser.red.service.persistence.management.v1.processor.utils.JSONDownloadFileTypeConverter; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DownloadFileType; -import com.iqser.red.service.persistence.service.v1.api.model.download.DownloadStatusValue; -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; -import org.hibernate.annotations.BatchSize; -import org.hibernate.annotations.Fetch; -import org.hibernate.annotations.FetchMode; - -import javax.persistence.*; import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; +import javax.persistence.Column; +import javax.persistence.Convert; +import javax.persistence.Entity; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; +import javax.persistence.FetchType; +import javax.persistence.Id; +import javax.persistence.ManyToMany; +import javax.persistence.ManyToOne; +import javax.persistence.Table; + +import org.hibernate.annotations.Fetch; +import org.hibernate.annotations.FetchMode; + +import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; +import com.iqser.red.service.persistence.management.v1.processor.utils.JSONDownloadFileTypeConverter; +import com.iqser.red.service.persistence.management.v1.processor.utils.JSONStoredFileInformationConverter; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DownloadFileType; +import com.iqser.red.service.persistence.service.v1.api.model.download.DownloadStatusValue; +import com.iqser.red.service.redaction.report.v1.api.model.StoredFileInformation; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.FieldDefaults; + @Data @Entity @Builder @AllArgsConstructor @NoArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE) @Table(name = "download_status") public class DownloadStatusEntity { @Id - private String storageId; + String storageId; @Column - private String userId; + String userId; @Column - private String filename; + String filename; @Column - private String mimeType; + String mimeType; @Column @Enumerated(EnumType.STRING) - private DownloadStatusValue status; + DownloadStatusValue status; @Column - private OffsetDateTime creationDate; + OffsetDateTime creationDate; @Column - private OffsetDateTime lastDownload; + OffsetDateTime lastDownload; @Column - private long fileSize; + long fileSize; @ManyToOne(fetch = FetchType.LAZY) - private DossierEntity dossier; + DossierEntity dossier; @ManyToMany @Fetch(FetchMode.SUBSELECT) - private List files = new ArrayList<>(); + List files = new ArrayList<>(); @Builder.Default @Column(columnDefinition = "text", name = "download_file_types") @Convert(converter = JSONDownloadFileTypeConverter.class) - private Set downloadFileTypes = new HashSet<>(); - + Set downloadFileTypes = new HashSet<>(); + @Builder.Default + @Column(columnDefinition = "text", name = "generated_reports_information") + @Convert(converter = JSONStoredFileInformationConverter.class) + Set generatedReportsInformation = new HashSet<>(); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DownloadStatusPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DownloadStatusPersistenceService.java index f20e066d6..0ba2843f6 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DownloadStatusPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DownloadStatusPersistenceService.java @@ -1,5 +1,15 @@ package com.iqser.red.service.persistence.management.v1.processor.service.persistence; +import java.time.OffsetDateTime; +import java.time.temporal.ChronoUnit; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import javax.transaction.Transactional; + +import org.springframework.stereotype.Service; + import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.download.DownloadStatusEntity; import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; @@ -7,16 +17,8 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DownloadFileType; import com.iqser.red.service.persistence.service.v1.api.model.download.DownloadStatusValue; -import lombok.RequiredArgsConstructor; -import org.springframework.stereotype.Service; -import javax.transaction.Transactional; -import java.time.OffsetDateTime; -import java.time.temporal.ChronoUnit; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; +import lombok.RequiredArgsConstructor; @Service @RequiredArgsConstructor @@ -76,6 +78,11 @@ public class DownloadStatusPersistenceService { } + public void updateStatusEntity(DownloadStatusEntity downloadStatusEntity) { + downloadStatusRepository.save(downloadStatusEntity); + } + + public List getStatusesByUser(String userId) { return downloadStatusRepository.findAllByUserId(userId); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/JSONStoredFileInformationConverter.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/JSONStoredFileInformationConverter.java new file mode 100644 index 000000000..794a66761 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/JSONStoredFileInformationConverter.java @@ -0,0 +1,42 @@ +package com.iqser.red.service.persistence.management.v1.processor.utils; + +import java.util.HashSet; +import java.util.Set; + +import javax.persistence.AttributeConverter; +import javax.persistence.Converter; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iqser.red.service.redaction.report.v1.api.model.StoredFileInformation; + +import lombok.SneakyThrows; + +@Converter +public class JSONStoredFileInformationConverter implements AttributeConverter, String> { + + private final ObjectMapper objectMapper = new ObjectMapper(); + + + @SneakyThrows + @Override + public String convertToDatabaseColumn(Set dataSet) { + + return objectMapper.writeValueAsString(dataSet); + } + + + @SneakyThrows + @Override + public Set convertToEntityAttribute(String data) { + + if (data == null) { + return new HashSet<>(); + } + TypeReference> typeRef = new TypeReference<>() { + }; + return objectMapper.readValue(data, typeRef); + + } + +} diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/DownloadPreparationService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/DownloadPreparationService.java index 875c8610e..20ef4da3d 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/DownloadPreparationService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/DownloadPreparationService.java @@ -1,6 +1,11 @@ package com.iqser.red.service.peristence.v1.server.service.download; +import java.io.IOException; +import java.util.Collection; +import java.util.LinkedList; +import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.stream.Collectors; import javax.transaction.Transactional; @@ -11,20 +16,16 @@ import org.springframework.stereotype.Service; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionMessage; -import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionRequest; +import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionResultDetail; +import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionResultMessage; import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionType; -import com.iqser.red.service.peristence.v1.server.client.RedactionClient; import com.iqser.red.service.peristence.v1.server.configuration.MessagingConfiguration; import com.iqser.red.service.peristence.v1.server.service.FileManagementStorageService; -import com.iqser.red.service.peristence.v1.server.service.RedactionLogService; import com.iqser.red.service.peristence.v1.server.utils.FileSystemBackedArchiver; -import com.iqser.red.service.persistence.management.v1.processor.client.PDFTronRedactionClient; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.ReportTemplateEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.download.DownloadStatusEntity; -import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DownloadStatusPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.NotificationPersistenceService; @@ -35,7 +36,7 @@ import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.do import com.iqser.red.service.persistence.service.v1.api.model.download.DownloadStatusValue; import com.iqser.red.service.redaction.report.v1.api.model.ReportResultMessage; import com.iqser.red.service.redaction.report.v1.api.model.StoredFileInformation; -import com.iqser.red.service.redaction.v1.model.RedactionLog; +import com.iqser.red.storage.commons.service.StorageService; import lombok.AccessLevel; import lombok.RequiredArgsConstructor; @@ -51,63 +52,66 @@ public class DownloadPreparationService { DownloadStatusPersistenceService downloadStatusPersistenceService; FileStatusPersistenceService fileStatusPersistenceService; FileManagementStorageService fileManagementStorageService; - RedactionClient redactionClient; - PDFTronRedactionClient pdfTronRedactionClient; ReportTemplatePersistenceService reportTemplatePersistenceService; - RedactionLogService redactionLogService; NotificationPersistenceService notificationPersistenceService; - DossierPersistenceService dossierPersistenceService; RabbitTemplate rabbitTemplate; ObjectMapper objectMapper; + StorageService storageService; @Transactional public void createDownload(ReportResultMessage reportResultMessage) throws JsonProcessingException { DownloadStatusEntity downloadStatus = downloadStatusPersistenceService.getStatus(reportResultMessage.getDownloadId()); + downloadStatus.setGeneratedReportsInformation(downloadStatus.getGeneratedReportsInformation()); + downloadStatusPersistenceService.updateStatusEntity(downloadStatus); + DossierEntity dossier = downloadStatus.getDossier(); - RedactionMessage message = RedactionMessage.builder().build(); - - for (String fileId : downloadStatus.getFiles().stream().map(FileEntity::getId).collect(Collectors.toUnmodifiableList())) { - for (DownloadFileType downloadFileType : downloadStatus.getDownloadFileTypes()) { - RedactionType redactionType = toRedactionType(downloadFileType); - if (redactionType == null) { - continue; - } - - message.getRedactionTypes().add(redactionType); - message.getFileIds().add(fileId); - message.getDossierIds().add(dossier.getId()); - } - } + RedactionMessage message = RedactionMessage.builder() + .dossierId(dossier.getId()) + .downloadId(reportResultMessage.getDownloadId()) + .redactionTypes(toPdfTronRedactionTypes(downloadStatus.getDownloadFileTypes())) + .fileIds(downloadStatus.getFiles().stream().map(FileEntity::getId).collect(Collectors.toList())) + .build(); rabbitTemplate.convertAndSend(MessagingConfiguration.PDFTRON_QUEUE, objectMapper.writeValueAsString(message)); } - private RedactionType toRedactionType(DownloadFileType downloadFileType) { + private List toPdfTronRedactionTypes(Collection downloadFileTypes) { - switch (downloadFileType) { - case REDACTED: - return RedactionType.REDACTED; - case PREVIEW: - return RedactionType.PREVIEW; - case DELTA_PREVIEW: - return RedactionType.DELTA; - case ORIGINAL: - default: - return null; + var result = new LinkedList(); + for (var downloadFileType : downloadFileTypes) { + switch (downloadFileType) { + case REDACTED: + result.add(RedactionType.REDACTED); + break; + case PREVIEW: + result.add(RedactionType.PREVIEW); + break; + case DELTA_PREVIEW: + result.add(RedactionType.DELTA); + break; + default: + // Other types don't need to be passed to pdf-tron + break; + } } + + return result; } - private void createDownloadArchive(ReportResultMessage reportResultMessage, DownloadStatusEntity downloadStatus, DossierEntity dossier) { + @Transactional + public void createDownload(RedactionResultMessage reportResultMessage) { + + DownloadStatusEntity downloadStatus = downloadStatusPersistenceService.getStatus(reportResultMessage.getDownloadId()); try (FileSystemBackedArchiver fileSystemBackedArchiver = new FileSystemBackedArchiver()) { - generateAndAddFiles(downloadStatus, fileSystemBackedArchiver); - addReports(reportResultMessage, fileSystemBackedArchiver); + generateAndAddFiles(downloadStatus, reportResultMessage, fileSystemBackedArchiver); + addReports(downloadStatus, fileSystemBackedArchiver); storeZipFile(downloadStatus, fileSystemBackedArchiver); downloadStatusPersistenceService.updateStatus(downloadStatus.getStorageId(), DownloadStatusValue.READY, fileSystemBackedArchiver.getContentLength()); @@ -116,13 +120,15 @@ public class DownloadPreparationService { .userId(downloadStatus.getUserId()) .issuerId(downloadStatus.getUserId()) .notificationType("DOWNLOAD_READY") - .target(Map.of("userId", downloadStatus.getUserId(), "dossierId", dossier.getId(), "downloadId", reportResultMessage.getDownloadId())) + .target(Map.of("userId", downloadStatus.getUserId(), // + "dossierId", downloadStatus.getDossier().getId(), // + "downloadId", downloadStatus.getStatus())) .build()); } } - private void generateAndAddFiles(DownloadStatusEntity downloadStatus, FileSystemBackedArchiver fileSystemBackedArchiver) { + private void generateAndAddFiles(DownloadStatusEntity downloadStatus, RedactionResultMessage reportResultMessage, FileSystemBackedArchiver fileSystemBackedArchiver) { int i = 1; long fileGenerationStart = System.currentTimeMillis(); @@ -133,23 +139,22 @@ public class DownloadPreparationService { FileEntity fileStatus = fileStatusPersistenceService.getStatus(fileId); byte[] original = fileManagementStorageService.getStoredObjectBytes(fileStatus.getDossierId(), fileId, FileType.ORIGIN); - RedactionLog redactionLog = getRedactionLog(fileStatus.getDossierId(), fileId, fileStatus.isExcluded()); - - var dossier = dossierPersistenceService.findByDossierId(fileStatus.getDossierId()); - for (DownloadFileType downloadFileType : downloadStatus.getDownloadFileTypes()) { if (downloadFileType.name().equals(DownloadFileType.ORIGINAL.name())) { fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel("Original", fileStatus.getFilename(), original)); } if (downloadFileType.name().equals(DownloadFileType.PREVIEW.name())) { - fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel("Preview", addSuffix(fileStatus.getFilename(), "highlighted"), getPreview(dossier.getId(), fileId, redactionLog, dossier.getDossierTemplateId(), dossier.getPreviewWatermarkId()))); + fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel("Preview", addSuffix(fileStatus.getFilename(), "highlighted"), // + getPreview(fileId, reportResultMessage.getRedactionResultDetails()))); } if (downloadFileType.name().equals(DownloadFileType.DELTA_PREVIEW.name())) { - fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel("Delta Preview", addSuffix(fileStatus.getFilename(), "delta_highlighted"), getDeltaPreview(dossier.getId(), fileId, redactionLog, dossier.getDossierTemplateId(), dossier.getPreviewWatermarkId()))); + fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel("Delta Preview", addSuffix(fileStatus.getFilename(), "delta_highlighted"), // + getDeltaPreview(fileId, reportResultMessage.getRedactionResultDetails()))); } if (downloadFileType.name().equals(DownloadFileType.REDACTED.name())) { - fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel("Redacted", addSuffix(fileStatus.getFilename(), "redacted"), getRedacted(dossier.getId(), fileId, redactionLog, dossier.getDossierTemplateId(), dossier.getWatermarkId()))); + fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel("Redacted", addSuffix(fileStatus.getFilename(), "redacted"), // + getRedacted(fileId, reportResultMessage.getRedactionResultDetails()))); } } log.info("Successfully added file {}/{} for downloadId {}, took {}", i, fileIds.size(), downloadStatus.getStorageId(), System.currentTimeMillis() - start); @@ -159,11 +164,46 @@ public class DownloadPreparationService { } - private void addReports(ReportResultMessage reportResultMessage, FileSystemBackedArchiver fileSystemBackedArchiver) { + private byte[] getRedacted(String fileId, List redactionResultDetails) { + + return getStoredFileBytes(fileId, redactionResultDetails, RedactionType.REDACTED); + } + + + private byte[] getStoredFileBytes(String fileId, List redactionResultDetails, RedactionType redactionType) { + + var redactionResultDetail = redactionResultDetails.stream() + .filter(rrd -> Objects.equals(fileId, rrd.getFileId()) && rrd.getRedactionType() == redactionType) + .findFirst(); + if (redactionResultDetail.isPresent()) { + try (var inputStream = storageService.getObject(redactionResultDetail.get().getStorageId()).getInputStream()) { + return inputStream.readAllBytes(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + throw new RuntimeException(String.format("Result for file %s of type %s not found", fileId, redactionType)); + } + + + private byte[] getDeltaPreview(String fileId, List redactionResultDetails) { + + return getStoredFileBytes(fileId, redactionResultDetails, RedactionType.DELTA); + } + + + private byte[] getPreview(String fileId, List redactionResultDetails) { + + return getStoredFileBytes(fileId, redactionResultDetails, RedactionType.PREVIEW); + } + + + private void addReports(DownloadStatusEntity downloadStatus, FileSystemBackedArchiver fileSystemBackedArchiver) { long addReportsStart = System.currentTimeMillis(); - for (StoredFileInformation storedFileInformation : reportResultMessage.getStoredFileInformation()) { + for (StoredFileInformation storedFileInformation : downloadStatus.getGeneratedReportsInformation()) { FileEntity fileStatus = null; @@ -176,8 +216,8 @@ public class DownloadPreparationService { fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(removeExtension(reportTemplate.getFileName()) + (reportTemplate.isMultiFileReport() ? " (multifile)" : ""), addSuffix(createFileName(fileStatus, reportTemplate), "justification"), report)); } - log.info("Successfully added {} reports for downloadId {}, took {}", reportResultMessage.getStoredFileInformation() - .size(), reportResultMessage.getDownloadId(), System.currentTimeMillis() - addReportsStart); + log.info("Successfully added {} reports for downloadId {}, took {}", downloadStatus.getGeneratedReportsInformation() + .size(), downloadStatus.getStorageId(), System.currentTimeMillis() - addReportsStart); } @@ -228,54 +268,4 @@ public class DownloadPreparationService { log.info("Successfully stored zip for downloadId {}, took {}", downloadStatus.getStorageId(), System.currentTimeMillis() - start); } - - private byte[] getRedacted(String dossierId, String fileId, RedactionLog redactionLog, String dossierTemplateId, Long watermarkId) { - - return pdfTronRedactionClient.redact(RedactionRequest.builder() - .dossierId(dossierId) - .fileId(fileId) - .dossierTemplateId(dossierTemplateId) - .redactionLog(redactionLog) - .watermarkId(watermarkId) - .build()).getDocument(); - } - - - private byte[] getPreview(String dossierId, String fileId, RedactionLog redactionLog, String dossierTemplateId, Long watermarkId) { - - return pdfTronRedactionClient.redactionPreview(RedactionRequest.builder() - .dossierId(dossierId) - .fileId(fileId) - .dossierTemplateId(dossierTemplateId) - .redactionLog(redactionLog) - .watermarkId(watermarkId) - .build()).getDocument(); - } - - - private byte[] getDeltaPreview(String dossierId, String fileId, RedactionLog redactionLog, String dossierTemplateId, Long watermarkId) { - - return pdfTronRedactionClient.redactionPreviewDiff(RedactionRequest.builder() - .dossierId(dossierId) - .fileId(fileId) - .dossierTemplateId(dossierTemplateId) - .redactionLog(redactionLog) - .watermarkId(watermarkId) - .build()).getDocument(); - } - - - private RedactionLog getRedactionLog(String dossierId, String fileId, boolean isExcluded) { - - if (isExcluded) { - return null; - } - try { - return redactionLogService.getRedactionLog(dossierId, fileId, true, false); - } catch (NotFoundException e) { - return null; - } - - } - } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/RedactionResultMessageReceiver.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/RedactionResultMessageReceiver.java new file mode 100644 index 000000000..8803ff7c4 --- /dev/null +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/RedactionResultMessageReceiver.java @@ -0,0 +1,33 @@ +package com.iqser.red.service.peristence.v1.server.service.download; + +import org.springframework.amqp.rabbit.annotation.RabbitHandler; +import org.springframework.amqp.rabbit.annotation.RabbitListener; +import org.springframework.stereotype.Service; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionResultMessage; +import com.iqser.red.service.peristence.v1.server.configuration.MessagingConfiguration; + +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; +import lombok.experimental.FieldDefaults; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +@RequiredArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true) +@RabbitListener(queues = MessagingConfiguration.REPORT_RESULT_QUEUE) +public class RedactionResultMessageReceiver { + ObjectMapper objectMapper; + + DownloadPreparationService downloadPreparationService; + + @RabbitHandler + public void receive(String in) throws JsonProcessingException { + + RedactionResultMessage redactionResultMessage = objectMapper.readValue(in, RedactionResultMessage.class); + downloadPreparationService.createDownload(redactionResultMessage); + } +} diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/34-add-reports-information-column.changelog.yaml b/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/34-add-reports-information-column.changelog.yaml new file mode 100644 index 000000000..42032274e --- /dev/null +++ b/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/34-add-reports-information-column.changelog.yaml @@ -0,0 +1,11 @@ +databaseChangeLog: + - changeSet: + id: add-reports-information-column + author: viktorseifert + changes: + - addColumn: + columns: + - column: + name: generated_reports_information + type: VARCHAR(2000) + tableName: download_status diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/db.changelog-master.yaml b/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/db.changelog-master.yaml index 4f2e1fbe9..e2382618f 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/db.changelog-master.yaml +++ b/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/db.changelog-master.yaml @@ -81,3 +81,5 @@ databaseChangeLog: file: db/changelog/33-add-file-processing-error-counter-column.changelog.yaml - include: file: db/changelog/sql/33-set-file-processing-error-counter.sql + - include: + file: db/changelog/34-add-reports-information-column.changelog.yaml From 80621cfab2e36a30e7b2448644065d7352d35dbc Mon Sep 17 00:00:00 2001 From: Viktor Seifert Date: Thu, 30 Jun 2022 17:34:51 +0200 Subject: [PATCH 4/9] RED-1098: Moved cleanup behavior to the correct location (after receiving the pdftron-results) --- .../download/DownloadPreparationService.java | 12 +++++++---- .../DownloadReportCleanupService.java | 17 ++++++++------- .../DownloadReportMessageReceiver.java | 21 ++++++++++++------- .../RedactionResultMessageReceiver.java | 7 ++++++- 4 files changed, 37 insertions(+), 20 deletions(-) diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/DownloadPreparationService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/DownloadPreparationService.java index 20ef4da3d..32d9bdca2 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/DownloadPreparationService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/DownloadPreparationService.java @@ -2,6 +2,7 @@ package com.iqser.red.service.peristence.v1.server.service.download; import java.io.IOException; import java.util.Collection; +import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -58,12 +59,14 @@ public class DownloadPreparationService { ObjectMapper objectMapper; StorageService storageService; + DownloadReportCleanupService downloadReportCleanupService; + @Transactional public void createDownload(ReportResultMessage reportResultMessage) throws JsonProcessingException { DownloadStatusEntity downloadStatus = downloadStatusPersistenceService.getStatus(reportResultMessage.getDownloadId()); - downloadStatus.setGeneratedReportsInformation(downloadStatus.getGeneratedReportsInformation()); + downloadStatus.setGeneratedReportsInformation(new HashSet<>(reportResultMessage.getStoredFileInformation())); downloadStatusPersistenceService.updateStatusEntity(downloadStatus); DossierEntity dossier = downloadStatus.getDossier(); @@ -75,6 +78,7 @@ public class DownloadPreparationService { .fileIds(downloadStatus.getFiles().stream().map(FileEntity::getId).collect(Collectors.toList())) .build(); + log.info("Sending redaction request for downloadId:{} to pdfton-redaction-queue", message.getDownloadId()); rabbitTemplate.convertAndSend(MessagingConfiguration.PDFTRON_QUEUE, objectMapper.writeValueAsString(message)); } @@ -125,6 +129,8 @@ public class DownloadPreparationService { "downloadId", downloadStatus.getStatus())) .build()); } + + downloadReportCleanupService.deleteTmpReportFiles(downloadStatus.getGeneratedReportsInformation()); } @@ -172,9 +178,7 @@ public class DownloadPreparationService { private byte[] getStoredFileBytes(String fileId, List redactionResultDetails, RedactionType redactionType) { - var redactionResultDetail = redactionResultDetails.stream() - .filter(rrd -> Objects.equals(fileId, rrd.getFileId()) && rrd.getRedactionType() == redactionType) - .findFirst(); + var redactionResultDetail = redactionResultDetails.stream().filter(rrd -> Objects.equals(fileId, rrd.getFileId()) && rrd.getRedactionType() == redactionType).findFirst(); if (redactionResultDetail.isPresent()) { try (var inputStream = storageService.getObject(redactionResultDetail.get().getStorageId()).getInputStream()) { return inputStream.readAllBytes(); diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/DownloadReportCleanupService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/DownloadReportCleanupService.java index 228d2620d..93ceac143 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/DownloadReportCleanupService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/DownloadReportCleanupService.java @@ -1,13 +1,16 @@ package com.iqser.red.service.peristence.v1.server.service.download; -import com.iqser.red.service.peristence.v1.server.service.FileManagementStorageService; -import com.iqser.red.service.redaction.report.v1.api.model.ReportResultMessage; -import com.iqser.red.service.redaction.report.v1.api.model.StoredFileInformation; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; +import java.util.Collection; + import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; +import com.iqser.red.service.peristence.v1.server.service.FileManagementStorageService; +import com.iqser.red.service.redaction.report.v1.api.model.StoredFileInformation; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + @Slf4j @Service @RequiredArgsConstructor @@ -16,9 +19,9 @@ public class DownloadReportCleanupService { private final FileManagementStorageService fileManagementStorageService; @Async - public void deleteTmpReportFiles(ReportResultMessage reportResultMessage) { + public void deleteTmpReportFiles(Collection fileInformationList) { - for (StoredFileInformation storedFileInformation : reportResultMessage.getStoredFileInformation()) { + for (StoredFileInformation storedFileInformation : fileInformationList) { fileManagementStorageService.deleteObject(storedFileInformation.getStorageId()); log.info("Deleted tmp report file {}", storedFileInformation.getStorageId()); } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/DownloadReportMessageReceiver.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/DownloadReportMessageReceiver.java index dc1b84244..d7f631ab1 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/DownloadReportMessageReceiver.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/DownloadReportMessageReceiver.java @@ -1,33 +1,38 @@ package com.iqser.red.service.peristence.v1.server.service.download; +import org.springframework.amqp.rabbit.annotation.RabbitHandler; +import org.springframework.amqp.rabbit.annotation.RabbitListener; +import org.springframework.stereotype.Service; + import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.iqser.red.service.peristence.v1.server.configuration.MessagingConfiguration; import com.iqser.red.service.redaction.report.v1.api.model.ReportResultMessage; + +import lombok.AccessLevel; import lombok.RequiredArgsConstructor; +import lombok.experimental.FieldDefaults; import lombok.extern.slf4j.Slf4j; -import org.springframework.amqp.rabbit.annotation.RabbitHandler; -import org.springframework.amqp.rabbit.annotation.RabbitListener; -import org.springframework.stereotype.Service; @Slf4j @Service @RequiredArgsConstructor @RabbitListener(queues = MessagingConfiguration.REPORT_RESULT_QUEUE) +@FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true) public class DownloadReportMessageReceiver { - private final ObjectMapper objectMapper; - private final DownloadPreparationService downloadPreparationService; - private final DownloadReportCleanupService downloadReportCleanupService; + ObjectMapper objectMapper; + DownloadPreparationService downloadPreparationService; @RabbitHandler public void receive(String in) throws JsonProcessingException { ReportResultMessage reportResultMessage = objectMapper.readValue(in, ReportResultMessage.class); + + log.info("Received request for userId:{} downloadId:{}", reportResultMessage.getUserId(), reportResultMessage.getDownloadId()); + downloadPreparationService.createDownload(reportResultMessage); - downloadReportCleanupService.deleteTmpReportFiles(reportResultMessage); - log.info("Successfully prepared download {}", reportResultMessage.getDownloadId()); } } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/RedactionResultMessageReceiver.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/RedactionResultMessageReceiver.java index 8803ff7c4..64cb85962 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/RedactionResultMessageReceiver.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/RedactionResultMessageReceiver.java @@ -20,14 +20,19 @@ import lombok.extern.slf4j.Slf4j; @FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true) @RabbitListener(queues = MessagingConfiguration.REPORT_RESULT_QUEUE) public class RedactionResultMessageReceiver { - ObjectMapper objectMapper; + ObjectMapper objectMapper; DownloadPreparationService downloadPreparationService; + @RabbitHandler public void receive(String in) throws JsonProcessingException { RedactionResultMessage redactionResultMessage = objectMapper.readValue(in, RedactionResultMessage.class); + + log.info("Received redaction results for downloadId:{}", redactionResultMessage.getDownloadId()); + downloadPreparationService.createDownload(redactionResultMessage); } + } From 78ae62955d65bfcad898e8819ca7835c2cb6510b Mon Sep 17 00:00:00 2001 From: Viktor Seifert Date: Thu, 30 Jun 2022 17:50:24 +0200 Subject: [PATCH 5/9] RED-1098: Corrected queue setting on pdftron-message-receiver --- .../server/service/download/RedactionResultMessageReceiver.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/RedactionResultMessageReceiver.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/RedactionResultMessageReceiver.java index 64cb85962..246b8a72c 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/RedactionResultMessageReceiver.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/RedactionResultMessageReceiver.java @@ -18,7 +18,7 @@ import lombok.extern.slf4j.Slf4j; @Service @RequiredArgsConstructor @FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true) -@RabbitListener(queues = MessagingConfiguration.REPORT_RESULT_QUEUE) +@RabbitListener(queues = MessagingConfiguration.PDFTRON_RESULT_QUEUE) public class RedactionResultMessageReceiver { ObjectMapper objectMapper; From 8435da351cf445080742548ad78f6be3bddf5e66 Mon Sep 17 00:00:00 2001 From: Viktor Seifert Date: Thu, 30 Jun 2022 17:58:38 +0200 Subject: [PATCH 6/9] RED-1098: Corrected typo in log-message --- .../v1/server/service/download/DownloadPreparationService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/DownloadPreparationService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/DownloadPreparationService.java index 32d9bdca2..0abf73232 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/DownloadPreparationService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/DownloadPreparationService.java @@ -78,7 +78,7 @@ public class DownloadPreparationService { .fileIds(downloadStatus.getFiles().stream().map(FileEntity::getId).collect(Collectors.toList())) .build(); - log.info("Sending redaction request for downloadId:{} to pdfton-redaction-queue", message.getDownloadId()); + log.info("Sending redaction request for downloadId:{} to pdftron-redaction-queue", message.getDownloadId()); rabbitTemplate.convertAndSend(MessagingConfiguration.PDFTRON_QUEUE, objectMapper.writeValueAsString(message)); } From 1c3f2e8174ce77ffd00e4e912c28aae67cc98ac7 Mon Sep 17 00:00:00 2001 From: Viktor Seifert Date: Fri, 1 Jul 2022 12:01:55 +0200 Subject: [PATCH 7/9] RED-1098: Implemented a receiver for dead messages from the pdftron-exchanges --- .../configuration/MessagingConfiguration.java | 23 +++++++- .../download/RedactionDlqMessageReceiver.java | 52 +++++++++++++++++++ 2 files changed, 73 insertions(+), 2 deletions(-) create mode 100644 persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/RedactionDlqMessageReceiver.java diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/configuration/MessagingConfiguration.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/configuration/MessagingConfiguration.java index bdfc6b53a..e213785b3 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/configuration/MessagingConfiguration.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/configuration/MessagingConfiguration.java @@ -49,6 +49,7 @@ public class MessagingConfiguration { public static final String PDF_2_IMAGE_DLQ = "pdf2image_dead_letter_queue"; public static final String PDFTRON_QUEUE = "pdftron_queue"; + public static final String PDFTRON_DLQ = "pdftron_dlq"; public static final String PDFTRON_RESULT_QUEUE = "pdftron_result_queue"; @@ -246,13 +247,31 @@ public class MessagingConfiguration { @Bean public Queue pdfTronQueue() { - return QueueBuilder.durable(PDFTRON_QUEUE).build(); + return QueueBuilder.durable(PDFTRON_QUEUE) + .withArgument("x-dead-letter-exchange", "") + .withArgument("x-dead-letter-routing-key", PDFTRON_DLQ) + .withArgument("x-max-priority", 2) + .maxPriority(2) + .build(); } + + @Bean + public Queue pdfTronDlq() { + + return QueueBuilder.durable(PDFTRON_DLQ).build(); + } + + @Bean public Queue pdfTronResultQueue() { - return QueueBuilder.durable(PDFTRON_RESULT_QUEUE).build(); + return QueueBuilder.durable(PDFTRON_RESULT_QUEUE) + .withArgument("x-dead-letter-exchange", "") + .withArgument("x-dead-letter-routing-key", PDFTRON_DLQ) + .withArgument("x-max-priority", 2) + .maxPriority(2) + .build(); } } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/RedactionDlqMessageReceiver.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/RedactionDlqMessageReceiver.java new file mode 100644 index 000000000..a1041a219 --- /dev/null +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/RedactionDlqMessageReceiver.java @@ -0,0 +1,52 @@ +package com.iqser.red.service.peristence.v1.server.service.download; + +import org.springframework.amqp.rabbit.annotation.RabbitHandler; +import org.springframework.amqp.rabbit.annotation.RabbitListener; +import org.springframework.stereotype.Service; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iqser.red.service.peristence.v1.server.configuration.MessagingConfiguration; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DownloadStatusPersistenceService; +import com.iqser.red.service.persistence.service.v1.api.model.download.DownloadStatusValue; + +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; +import lombok.experimental.FieldDefaults; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +@RequiredArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true) +@RabbitListener(queues = MessagingConfiguration.PDFTRON_DLQ) +public class RedactionDlqMessageReceiver { + + private static final String LINE_SEPARATOR = System.lineSeparator(); + + ObjectMapper objectMapper; + DownloadStatusPersistenceService downloadStatusPersistenceService; + + + @RabbitHandler + public void receive(String in) throws JsonProcessingException { + + // Since we receive different message types here, we do not convert to an object here; + // We just assume that the message contains a downloadId. + JsonNode jsonNode = objectMapper.readTree(in); + final String downloadId; + try { + downloadId = jsonNode.findValue("downloadId").asText(); + } catch (Exception e) { + log.warn("Received a message in the " + MessagingConfiguration.PDFTRON_DLQ + " that contains no downloadId" + LINE_SEPARATOR + "{}", jsonNode.asText()); + throw new RuntimeException(e); + } + + log.info("Received a dead message with downloadId:{}, updating the download as failed", downloadId); + + downloadStatusPersistenceService.updateStatus(downloadId, DownloadStatusValue.FAILED); + + } + +} From 79d99eb18ef0a87ae9f4737900f5ef7442126604 Mon Sep 17 00:00:00 2001 From: Viktor Seifert Date: Fri, 1 Jul 2022 13:59:08 +0200 Subject: [PATCH 8/9] RED-1098: Updated pdftron-redaction-service version to include the newly created DTO's --- persistence-service-v1/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/persistence-service-v1/pom.xml b/persistence-service-v1/pom.xml index d53903a98..58d00c915 100755 --- a/persistence-service-v1/pom.xml +++ b/persistence-service-v1/pom.xml @@ -27,7 +27,7 @@ 3.114.0 2.36.0 - 3.111.0 + 3.112.0 3.47.0 From afc837163677aaa33fe1c8c25f7b52c39dd41a66 Mon Sep 17 00:00:00 2001 From: Viktor Seifert Date: Fri, 1 Jul 2022 15:31:06 +0200 Subject: [PATCH 9/9] RED-1098: Added files generated by pdftron to the list of files that are deleted when the download archive has been created --- .../service/download/DownloadPreparationService.java | 9 ++++++++- .../service/download/DownloadReportCleanupService.java | 9 ++++----- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/DownloadPreparationService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/DownloadPreparationService.java index 0abf73232..b8c29fe47 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/DownloadPreparationService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/DownloadPreparationService.java @@ -130,7 +130,14 @@ public class DownloadPreparationService { .build()); } - downloadReportCleanupService.deleteTmpReportFiles(downloadStatus.getGeneratedReportsInformation()); + downloadReportCleanupService.deleteTmpReportFiles(downloadStatus.getGeneratedReportsInformation() + .stream() + .map(StoredFileInformation::getStorageId) + .collect(Collectors.toSet())); + downloadReportCleanupService.deleteTmpReportFiles(reportResultMessage.getRedactionResultDetails() + .stream() + .map(RedactionResultDetail::getStorageId) + .collect(Collectors.toSet())); } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/DownloadReportCleanupService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/DownloadReportCleanupService.java index 93ceac143..415e4baf5 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/DownloadReportCleanupService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/DownloadReportCleanupService.java @@ -6,7 +6,6 @@ import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; import com.iqser.red.service.peristence.v1.server.service.FileManagementStorageService; -import com.iqser.red.service.redaction.report.v1.api.model.StoredFileInformation; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -19,11 +18,11 @@ public class DownloadReportCleanupService { private final FileManagementStorageService fileManagementStorageService; @Async - public void deleteTmpReportFiles(Collection fileInformationList) { + public void deleteTmpReportFiles(Collection storageIds) { - for (StoredFileInformation storedFileInformation : fileInformationList) { - fileManagementStorageService.deleteObject(storedFileInformation.getStorageId()); - log.info("Deleted tmp report file {}", storedFileInformation.getStorageId()); + for (String storageId : storageIds) { + fileManagementStorageService.deleteObject(storageId); + log.info("Deleted tmp report file {}", storageId); } } }