diff --git a/persistence-service-v1/persistence-service-processor-v1/pom.xml b/persistence-service-v1/persistence-service-processor-v1/pom.xml index 907c93920..ed3b3ddb8 100644 --- a/persistence-service-v1/persistence-service-processor-v1/pom.xml +++ b/persistence-service-v1/persistence-service-processor-v1/pom.xml @@ -100,5 +100,9 @@ guava compile + + com.iqser.red.service + redaction-report-service-api-v1 + diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/download/DownloadStatusEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/download/DownloadStatusEntity.java index 460568f75..3b17c81de 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/download/DownloadStatusEntity.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/download/DownloadStatusEntity.java @@ -1,62 +1,81 @@ package com.iqser.red.service.persistence.management.v1.processor.entity.download; -import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity; -import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; -import com.iqser.red.service.persistence.management.v1.processor.utils.JSONDownloadFileTypeConverter; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DownloadFileType; -import com.iqser.red.service.persistence.service.v1.api.model.download.DownloadStatusValue; -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; -import org.hibernate.annotations.BatchSize; -import org.hibernate.annotations.Fetch; -import org.hibernate.annotations.FetchMode; - -import javax.persistence.*; import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; +import javax.persistence.Column; +import javax.persistence.Convert; +import javax.persistence.Entity; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; +import javax.persistence.FetchType; +import javax.persistence.Id; +import javax.persistence.ManyToMany; +import javax.persistence.ManyToOne; +import javax.persistence.Table; + +import org.hibernate.annotations.Fetch; +import org.hibernate.annotations.FetchMode; + +import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; +import com.iqser.red.service.persistence.management.v1.processor.utils.JSONDownloadFileTypeConverter; +import com.iqser.red.service.persistence.management.v1.processor.utils.JSONStoredFileInformationConverter; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DownloadFileType; +import com.iqser.red.service.persistence.service.v1.api.model.download.DownloadStatusValue; +import com.iqser.red.service.redaction.report.v1.api.model.StoredFileInformation; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.FieldDefaults; + @Data @Entity @Builder @AllArgsConstructor @NoArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE) @Table(name = "download_status") public class DownloadStatusEntity { @Id - private String storageId; + String storageId; @Column - private String userId; + String userId; @Column - private String filename; + String filename; @Column - private String mimeType; + String mimeType; @Column @Enumerated(EnumType.STRING) - private DownloadStatusValue status; + DownloadStatusValue status; @Column - private OffsetDateTime creationDate; + OffsetDateTime creationDate; @Column - private OffsetDateTime lastDownload; + OffsetDateTime lastDownload; @Column - private long fileSize; + long fileSize; @ManyToOne(fetch = FetchType.LAZY) - private DossierEntity dossier; + DossierEntity dossier; @ManyToMany @Fetch(FetchMode.SUBSELECT) - private List files = new ArrayList<>(); + List files = new ArrayList<>(); @Builder.Default @Column(columnDefinition = "text", name = "download_file_types") @Convert(converter = JSONDownloadFileTypeConverter.class) - private Set downloadFileTypes = new HashSet<>(); - + Set downloadFileTypes = new HashSet<>(); + @Builder.Default + @Column(columnDefinition = "text", name = "generated_reports_information") + @Convert(converter = JSONStoredFileInformationConverter.class) + Set generatedReportsInformation = new HashSet<>(); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DownloadStatusPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DownloadStatusPersistenceService.java index f20e066d6..0ba2843f6 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DownloadStatusPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DownloadStatusPersistenceService.java @@ -1,5 +1,15 @@ package com.iqser.red.service.persistence.management.v1.processor.service.persistence; +import java.time.OffsetDateTime; +import java.time.temporal.ChronoUnit; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import javax.transaction.Transactional; + +import org.springframework.stereotype.Service; + import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.download.DownloadStatusEntity; import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; @@ -7,16 +17,8 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DownloadFileType; import com.iqser.red.service.persistence.service.v1.api.model.download.DownloadStatusValue; -import lombok.RequiredArgsConstructor; -import org.springframework.stereotype.Service; -import javax.transaction.Transactional; -import java.time.OffsetDateTime; -import java.time.temporal.ChronoUnit; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; +import lombok.RequiredArgsConstructor; @Service @RequiredArgsConstructor @@ -76,6 +78,11 @@ public class DownloadStatusPersistenceService { } + public void updateStatusEntity(DownloadStatusEntity downloadStatusEntity) { + downloadStatusRepository.save(downloadStatusEntity); + } + + public List getStatusesByUser(String userId) { return downloadStatusRepository.findAllByUserId(userId); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/JSONStoredFileInformationConverter.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/JSONStoredFileInformationConverter.java new file mode 100644 index 000000000..794a66761 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/JSONStoredFileInformationConverter.java @@ -0,0 +1,42 @@ +package com.iqser.red.service.persistence.management.v1.processor.utils; + +import java.util.HashSet; +import java.util.Set; + +import javax.persistence.AttributeConverter; +import javax.persistence.Converter; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iqser.red.service.redaction.report.v1.api.model.StoredFileInformation; + +import lombok.SneakyThrows; + +@Converter +public class JSONStoredFileInformationConverter implements AttributeConverter, String> { + + private final ObjectMapper objectMapper = new ObjectMapper(); + + + @SneakyThrows + @Override + public String convertToDatabaseColumn(Set dataSet) { + + return objectMapper.writeValueAsString(dataSet); + } + + + @SneakyThrows + @Override + public Set convertToEntityAttribute(String data) { + + if (data == null) { + return new HashSet<>(); + } + TypeReference> typeRef = new TypeReference<>() { + }; + return objectMapper.readValue(data, typeRef); + + } + +} diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/configuration/MessagingConfiguration.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/configuration/MessagingConfiguration.java index 2abc99f61..e213785b3 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/configuration/MessagingConfiguration.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/configuration/MessagingConfiguration.java @@ -48,6 +48,10 @@ public class MessagingConfiguration { public static final String PDF_2_IMAGE_RESPONSE_QUEUE = "pdf2image_response_queue"; public static final String PDF_2_IMAGE_DLQ = "pdf2image_dead_letter_queue"; + public static final String PDFTRON_QUEUE = "pdftron_queue"; + public static final String PDFTRON_DLQ = "pdftron_dlq"; + public static final String PDFTRON_RESULT_QUEUE = "pdftron_result_queue"; + @Bean public Queue nerRequestQueue() { @@ -59,10 +63,7 @@ public class MessagingConfiguration { @Bean public Queue nerResponseQueue() { - return QueueBuilder.durable(NER_SERVICE_RESPONSE_QUEUE) - .withArgument("x-dead-letter-exchange", "") - .withArgument("x-dead-letter-routing-key", NER_SERVICE_DLQ) - .build(); + return QueueBuilder.durable(NER_SERVICE_RESPONSE_QUEUE).withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", NER_SERVICE_DLQ).build(); } @@ -76,18 +77,14 @@ public class MessagingConfiguration { @Bean public Queue imageRequestQueue() { - return QueueBuilder.durable(IMAGE_SERVICE_QUEUE) - .withArgument("x-dead-letter-exchange", "") - .withArgument("x-dead-letter-routing-key", IMAGE_SERVICE_DLQ) - .build(); + return QueueBuilder.durable(IMAGE_SERVICE_QUEUE).withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", IMAGE_SERVICE_DLQ).build(); } @Bean public Queue imageResponseQueue() { - return QueueBuilder.durable(IMAGE_SERVICE_RESPONSE_QUEUE) - .withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", IMAGE_SERVICE_DLQ).build(); + return QueueBuilder.durable(IMAGE_SERVICE_RESPONSE_QUEUE).withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", IMAGE_SERVICE_DLQ).build(); } @@ -152,10 +149,7 @@ public class MessagingConfiguration { @Bean public Queue downloadQueue() { - return QueueBuilder.durable(DOWNLOAD_QUEUE) - .withArgument("x-dead-letter-exchange", "") - .withArgument("x-dead-letter-routing-key", DOWNLOAD_DLQ) - .build(); + return QueueBuilder.durable(DOWNLOAD_QUEUE).withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", DOWNLOAD_DLQ).build(); } @@ -188,10 +182,7 @@ public class MessagingConfiguration { @Bean public Queue reportResultQueue() { - return QueueBuilder.durable(REPORT_RESULT_QUEUE) - .withArgument("x-dead-letter-exchange", "") - .withArgument("x-dead-letter-routing-key", REPORT_RESULT_DLQ) - .build(); + return QueueBuilder.durable(REPORT_RESULT_QUEUE).withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", REPORT_RESULT_DLQ).build(); } @@ -205,11 +196,7 @@ public class MessagingConfiguration { @Bean public Queue indexingQueue() { - return QueueBuilder.durable(INDEXING_QUEUE) - .withArgument("x-dead-letter-exchange", "") - .withArgument("x-dead-letter-routing-key", INDEXING_DQL) - .maxPriority(2) - .build(); + return QueueBuilder.durable(INDEXING_QUEUE).withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", INDEXING_DQL).maxPriority(2).build(); } @@ -237,6 +224,7 @@ public class MessagingConfiguration { return QueueBuilder.durable(DELETE_FROM_INDEX_DLQ).build(); } + @Bean public Queue preprocessingQueue() { @@ -255,4 +243,35 @@ public class MessagingConfiguration { return QueueBuilder.durable(PRE_PROCESSING_DLQ).build(); } + + @Bean + public Queue pdfTronQueue() { + + return QueueBuilder.durable(PDFTRON_QUEUE) + .withArgument("x-dead-letter-exchange", "") + .withArgument("x-dead-letter-routing-key", PDFTRON_DLQ) + .withArgument("x-max-priority", 2) + .maxPriority(2) + .build(); + } + + + @Bean + public Queue pdfTronDlq() { + + return QueueBuilder.durable(PDFTRON_DLQ).build(); + } + + + @Bean + public Queue pdfTronResultQueue() { + + return QueueBuilder.durable(PDFTRON_RESULT_QUEUE) + .withArgument("x-dead-letter-exchange", "") + .withArgument("x-dead-letter-routing-key", PDFTRON_DLQ) + .withArgument("x-max-priority", 2) + .maxPriority(2) + .build(); + } + } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/DownloadPreparationService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/DownloadPreparationService.java index fc98e375f..b8c29fe47 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/DownloadPreparationService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/DownloadPreparationService.java @@ -1,57 +1,121 @@ package com.iqser.red.service.peristence.v1.server.service.download; -import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionRequest; -import com.iqser.red.service.peristence.v1.server.client.RedactionClient; +import java.io.IOException; +import java.util.Collection; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +import javax.transaction.Transactional; + +import org.springframework.amqp.rabbit.core.RabbitTemplate; +import org.springframework.stereotype.Service; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionMessage; +import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionResultDetail; +import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionResultMessage; +import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionType; +import com.iqser.red.service.peristence.v1.server.configuration.MessagingConfiguration; import com.iqser.red.service.peristence.v1.server.service.FileManagementStorageService; -import com.iqser.red.service.peristence.v1.server.service.RedactionLogService; import com.iqser.red.service.peristence.v1.server.utils.FileSystemBackedArchiver; -import com.iqser.red.service.persistence.management.v1.processor.client.PDFTronRedactionClient; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.ReportTemplateEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.download.DownloadStatusEntity; -import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.*; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DownloadStatusPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.NotificationPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ReportTemplatePersistenceService; import com.iqser.red.service.persistence.service.v1.api.model.audit.AddNotificationRequest; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DownloadFileType; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.persistence.service.v1.api.model.download.DownloadStatusValue; import com.iqser.red.service.redaction.report.v1.api.model.ReportResultMessage; import com.iqser.red.service.redaction.report.v1.api.model.StoredFileInformation; -import com.iqser.red.service.redaction.v1.model.RedactionLog; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Service; +import com.iqser.red.storage.commons.service.StorageService; -import javax.transaction.Transactional; -import java.util.Map; -import java.util.stream.Collectors; +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; +import lombok.experimental.FieldDefaults; +import lombok.extern.slf4j.Slf4j; @Slf4j @Service @RequiredArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true) public class DownloadPreparationService { - private final DownloadStatusPersistenceService downloadStatusPersistenceService; - private final FileStatusPersistenceService fileStatusPersistenceService; - private final FileManagementStorageService fileManagementStorageService; - private final RedactionClient redactionClient; - private final PDFTronRedactionClient pdfTronRedactionClient; - private final ReportTemplatePersistenceService reportTemplatePersistenceService; - private final RedactionLogService redactionLogService; - private final NotificationPersistenceService notificationPersistenceService; - private final DossierPersistenceService dossierPersistenceService; + DownloadStatusPersistenceService downloadStatusPersistenceService; + FileStatusPersistenceService fileStatusPersistenceService; + FileManagementStorageService fileManagementStorageService; + ReportTemplatePersistenceService reportTemplatePersistenceService; + NotificationPersistenceService notificationPersistenceService; + RabbitTemplate rabbitTemplate; + ObjectMapper objectMapper; + StorageService storageService; + + DownloadReportCleanupService downloadReportCleanupService; + @Transactional - public void createDownload(ReportResultMessage reportResultMessage) { + public void createDownload(ReportResultMessage reportResultMessage) throws JsonProcessingException { DownloadStatusEntity downloadStatus = downloadStatusPersistenceService.getStatus(reportResultMessage.getDownloadId()); + downloadStatus.setGeneratedReportsInformation(new HashSet<>(reportResultMessage.getStoredFileInformation())); + downloadStatusPersistenceService.updateStatusEntity(downloadStatus); + DossierEntity dossier = downloadStatus.getDossier(); + RedactionMessage message = RedactionMessage.builder() + .dossierId(dossier.getId()) + .downloadId(reportResultMessage.getDownloadId()) + .redactionTypes(toPdfTronRedactionTypes(downloadStatus.getDownloadFileTypes())) + .fileIds(downloadStatus.getFiles().stream().map(FileEntity::getId).collect(Collectors.toList())) + .build(); + + log.info("Sending redaction request for downloadId:{} to pdftron-redaction-queue", message.getDownloadId()); + rabbitTemplate.convertAndSend(MessagingConfiguration.PDFTRON_QUEUE, objectMapper.writeValueAsString(message)); + } + + + private List toPdfTronRedactionTypes(Collection downloadFileTypes) { + + var result = new LinkedList(); + for (var downloadFileType : downloadFileTypes) { + switch (downloadFileType) { + case REDACTED: + result.add(RedactionType.REDACTED); + break; + case PREVIEW: + result.add(RedactionType.PREVIEW); + break; + case DELTA_PREVIEW: + result.add(RedactionType.DELTA); + break; + default: + // Other types don't need to be passed to pdf-tron + break; + } + } + + return result; + } + + + @Transactional + public void createDownload(RedactionResultMessage reportResultMessage) { + + DownloadStatusEntity downloadStatus = downloadStatusPersistenceService.getStatus(reportResultMessage.getDownloadId()); + try (FileSystemBackedArchiver fileSystemBackedArchiver = new FileSystemBackedArchiver()) { - generateAndAddFiles(downloadStatus, fileSystemBackedArchiver); - addReports(reportResultMessage, fileSystemBackedArchiver); + generateAndAddFiles(downloadStatus, reportResultMessage, fileSystemBackedArchiver); + addReports(downloadStatus, fileSystemBackedArchiver); storeZipFile(downloadStatus, fileSystemBackedArchiver); downloadStatusPersistenceService.updateStatus(downloadStatus.getStorageId(), DownloadStatusValue.READY, fileSystemBackedArchiver.getContentLength()); @@ -60,13 +124,24 @@ public class DownloadPreparationService { .userId(downloadStatus.getUserId()) .issuerId(downloadStatus.getUserId()) .notificationType("DOWNLOAD_READY") - .target(Map.of("userId", downloadStatus.getUserId(), "dossierId", dossier.getId(), "downloadId", reportResultMessage.getDownloadId())) + .target(Map.of("userId", downloadStatus.getUserId(), // + "dossierId", downloadStatus.getDossier().getId(), // + "downloadId", downloadStatus.getStatus())) .build()); } + + downloadReportCleanupService.deleteTmpReportFiles(downloadStatus.getGeneratedReportsInformation() + .stream() + .map(StoredFileInformation::getStorageId) + .collect(Collectors.toSet())); + downloadReportCleanupService.deleteTmpReportFiles(reportResultMessage.getRedactionResultDetails() + .stream() + .map(RedactionResultDetail::getStorageId) + .collect(Collectors.toSet())); } - private void generateAndAddFiles(DownloadStatusEntity downloadStatus, FileSystemBackedArchiver fileSystemBackedArchiver) { + private void generateAndAddFiles(DownloadStatusEntity downloadStatus, RedactionResultMessage reportResultMessage, FileSystemBackedArchiver fileSystemBackedArchiver) { int i = 1; long fileGenerationStart = System.currentTimeMillis(); @@ -77,42 +152,69 @@ public class DownloadPreparationService { FileEntity fileStatus = fileStatusPersistenceService.getStatus(fileId); byte[] original = fileManagementStorageService.getStoredObjectBytes(fileStatus.getDossierId(), fileId, FileType.ORIGIN); - RedactionLog redactionLog = getRedactionLog(fileStatus.getDossierId(), fileId, fileStatus.isExcluded()); - - var dossier = dossierPersistenceService.findByDossierId(fileStatus.getDossierId()); - for (DownloadFileType downloadFileType : downloadStatus.getDownloadFileTypes()) { if (downloadFileType.name().equals(DownloadFileType.ORIGINAL.name())) { fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel("Original", fileStatus.getFilename(), original)); } if (downloadFileType.name().equals(DownloadFileType.PREVIEW.name())) { - fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel("Preview", addSuffix(fileStatus.getFilename(), "highlighted"), - getPreview(dossier.getId(), fileId, redactionLog, dossier.getDossierTemplateId(), dossier.getPreviewWatermarkId()))); + fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel("Preview", addSuffix(fileStatus.getFilename(), "highlighted"), // + getPreview(fileId, reportResultMessage.getRedactionResultDetails()))); } if (downloadFileType.name().equals(DownloadFileType.DELTA_PREVIEW.name())) { - fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel("Delta Preview", addSuffix(fileStatus.getFilename(), "delta_highlighted"), - getDeltaPreview(dossier.getId(), fileId, redactionLog, dossier.getDossierTemplateId(), dossier.getPreviewWatermarkId()))); + fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel("Delta Preview", addSuffix(fileStatus.getFilename(), "delta_highlighted"), // + getDeltaPreview(fileId, reportResultMessage.getRedactionResultDetails()))); } if (downloadFileType.name().equals(DownloadFileType.REDACTED.name())) { - fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel("Redacted", addSuffix(fileStatus.getFilename(), "redacted"), - getRedacted(dossier.getId(), fileId, redactionLog, dossier.getDossierTemplateId(), dossier.getWatermarkId()))); + fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel("Redacted", addSuffix(fileStatus.getFilename(), "redacted"), // + getRedacted(fileId, reportResultMessage.getRedactionResultDetails()))); } } - log.info("Successfully added file {}/{} for downloadId {}, took {}", i, fileIds - .size(), downloadStatus.getStorageId(), System.currentTimeMillis() - start); + log.info("Successfully added file {}/{} for downloadId {}, took {}", i, fileIds.size(), downloadStatus.getStorageId(), System.currentTimeMillis() - start); i++; } log.info("Successfully added {} files for downloadId {}, took {}", fileIds, downloadStatus.getStorageId(), System.currentTimeMillis() - fileGenerationStart); } - private void addReports(ReportResultMessage reportResultMessage, FileSystemBackedArchiver fileSystemBackedArchiver) { + private byte[] getRedacted(String fileId, List redactionResultDetails) { + + return getStoredFileBytes(fileId, redactionResultDetails, RedactionType.REDACTED); + } + + + private byte[] getStoredFileBytes(String fileId, List redactionResultDetails, RedactionType redactionType) { + + var redactionResultDetail = redactionResultDetails.stream().filter(rrd -> Objects.equals(fileId, rrd.getFileId()) && rrd.getRedactionType() == redactionType).findFirst(); + if (redactionResultDetail.isPresent()) { + try (var inputStream = storageService.getObject(redactionResultDetail.get().getStorageId()).getInputStream()) { + return inputStream.readAllBytes(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + throw new RuntimeException(String.format("Result for file %s of type %s not found", fileId, redactionType)); + } + + + private byte[] getDeltaPreview(String fileId, List redactionResultDetails) { + + return getStoredFileBytes(fileId, redactionResultDetails, RedactionType.DELTA); + } + + + private byte[] getPreview(String fileId, List redactionResultDetails) { + + return getStoredFileBytes(fileId, redactionResultDetails, RedactionType.PREVIEW); + } + + + private void addReports(DownloadStatusEntity downloadStatus, FileSystemBackedArchiver fileSystemBackedArchiver) { long addReportsStart = System.currentTimeMillis(); - for (StoredFileInformation storedFileInformation : reportResultMessage.getStoredFileInformation()) { - + for (StoredFileInformation storedFileInformation : downloadStatus.getGeneratedReportsInformation()) { FileEntity fileStatus = null; @@ -122,14 +224,14 @@ public class DownloadPreparationService { ReportTemplateEntity reportTemplate = reportTemplatePersistenceService.find(storedFileInformation.getTemplateId()); byte[] report = fileManagementStorageService.getStoredObjectBytes(storedFileInformation.getStorageId()); - fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(removeExtension(reportTemplate.getFileName()) + (reportTemplate.isMultiFileReport() ? " (multifile)" : ""), - addSuffix(createFileName(fileStatus, reportTemplate), "justification"), report)); + fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(removeExtension(reportTemplate.getFileName()) + (reportTemplate.isMultiFileReport() ? " (multifile)" : ""), addSuffix(createFileName(fileStatus, reportTemplate), "justification"), report)); } - log.info("Successfully added {} reports for downloadId {}, took {}", reportResultMessage.getStoredFileInformation() - .size(), reportResultMessage.getDownloadId(), System.currentTimeMillis() - addReportsStart); + log.info("Successfully added {} reports for downloadId {}, took {}", downloadStatus.getGeneratedReportsInformation() + .size(), downloadStatus.getStorageId(), System.currentTimeMillis() - addReportsStart); } + private String createFileName(FileEntity fileStatus, ReportTemplateEntity reportTemplate) { if (fileStatus != null) { @@ -139,7 +241,9 @@ public class DownloadPreparationService { } } + private String getExtension(String fileName) { + var index = fileName.lastIndexOf("."); if (index > 0) { return fileName.substring(index); @@ -148,7 +252,9 @@ public class DownloadPreparationService { } } + private String removeExtension(String fileName) { + var index = fileName.lastIndexOf("."); if (index > 0) { return fileName.substring(0, index); @@ -159,6 +265,7 @@ public class DownloadPreparationService { private String addSuffix(String filename, String suffix) { + var oldExtension = getExtension(filename); return removeExtension(filename) + "_" + suffix + oldExtension; @@ -172,54 +279,4 @@ public class DownloadPreparationService { log.info("Successfully stored zip for downloadId {}, took {}", downloadStatus.getStorageId(), System.currentTimeMillis() - start); } - private byte[] getRedacted(String dossierId, String fileId, RedactionLog redactionLog, String dossierTemplateId, - Long watermarkId) { - - return pdfTronRedactionClient.redact(RedactionRequest.builder() - .dossierId(dossierId) - .fileId(fileId) - .dossierTemplateId(dossierTemplateId) - .redactionLog(redactionLog) - .watermarkId(watermarkId) - .build()).getDocument(); - } - - - private byte[] getPreview(String dossierId, String fileId, RedactionLog redactionLog, String dossierTemplateId, Long watermarkId) { - - return pdfTronRedactionClient.redactionPreview(RedactionRequest.builder() - .dossierId(dossierId) - .fileId(fileId) - .dossierTemplateId(dossierTemplateId) - .redactionLog(redactionLog) - .watermarkId(watermarkId) - .build()).getDocument(); - } - - - private byte[] getDeltaPreview(String dossierId, String fileId, RedactionLog redactionLog, String dossierTemplateId, Long watermarkId) { - - return pdfTronRedactionClient.redactionPreviewDiff(RedactionRequest.builder() - .dossierId(dossierId) - .fileId(fileId) - .dossierTemplateId(dossierTemplateId) - .redactionLog(redactionLog) - .watermarkId(watermarkId) - .build()).getDocument(); - } - - - private RedactionLog getRedactionLog(String dossierId, String fileId, boolean isExcluded) { - - if (isExcluded) { - return null; - } - try { - return redactionLogService.getRedactionLog(dossierId, fileId, true, false); - } catch (NotFoundException e) { - return null; - } - - } - } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/DownloadReportCleanupService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/DownloadReportCleanupService.java index 228d2620d..415e4baf5 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/DownloadReportCleanupService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/DownloadReportCleanupService.java @@ -1,13 +1,15 @@ package com.iqser.red.service.peristence.v1.server.service.download; -import com.iqser.red.service.peristence.v1.server.service.FileManagementStorageService; -import com.iqser.red.service.redaction.report.v1.api.model.ReportResultMessage; -import com.iqser.red.service.redaction.report.v1.api.model.StoredFileInformation; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; +import java.util.Collection; + import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; +import com.iqser.red.service.peristence.v1.server.service.FileManagementStorageService; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + @Slf4j @Service @RequiredArgsConstructor @@ -16,11 +18,11 @@ public class DownloadReportCleanupService { private final FileManagementStorageService fileManagementStorageService; @Async - public void deleteTmpReportFiles(ReportResultMessage reportResultMessage) { + public void deleteTmpReportFiles(Collection storageIds) { - for (StoredFileInformation storedFileInformation : reportResultMessage.getStoredFileInformation()) { - fileManagementStorageService.deleteObject(storedFileInformation.getStorageId()); - log.info("Deleted tmp report file {}", storedFileInformation.getStorageId()); + for (String storageId : storageIds) { + fileManagementStorageService.deleteObject(storageId); + log.info("Deleted tmp report file {}", storageId); } } } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/DownloadReportMessageReceiver.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/DownloadReportMessageReceiver.java index dc1b84244..d7f631ab1 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/DownloadReportMessageReceiver.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/DownloadReportMessageReceiver.java @@ -1,33 +1,38 @@ package com.iqser.red.service.peristence.v1.server.service.download; +import org.springframework.amqp.rabbit.annotation.RabbitHandler; +import org.springframework.amqp.rabbit.annotation.RabbitListener; +import org.springframework.stereotype.Service; + import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.iqser.red.service.peristence.v1.server.configuration.MessagingConfiguration; import com.iqser.red.service.redaction.report.v1.api.model.ReportResultMessage; + +import lombok.AccessLevel; import lombok.RequiredArgsConstructor; +import lombok.experimental.FieldDefaults; import lombok.extern.slf4j.Slf4j; -import org.springframework.amqp.rabbit.annotation.RabbitHandler; -import org.springframework.amqp.rabbit.annotation.RabbitListener; -import org.springframework.stereotype.Service; @Slf4j @Service @RequiredArgsConstructor @RabbitListener(queues = MessagingConfiguration.REPORT_RESULT_QUEUE) +@FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true) public class DownloadReportMessageReceiver { - private final ObjectMapper objectMapper; - private final DownloadPreparationService downloadPreparationService; - private final DownloadReportCleanupService downloadReportCleanupService; + ObjectMapper objectMapper; + DownloadPreparationService downloadPreparationService; @RabbitHandler public void receive(String in) throws JsonProcessingException { ReportResultMessage reportResultMessage = objectMapper.readValue(in, ReportResultMessage.class); + + log.info("Received request for userId:{} downloadId:{}", reportResultMessage.getUserId(), reportResultMessage.getDownloadId()); + downloadPreparationService.createDownload(reportResultMessage); - downloadReportCleanupService.deleteTmpReportFiles(reportResultMessage); - log.info("Successfully prepared download {}", reportResultMessage.getDownloadId()); } } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/RedactionDlqMessageReceiver.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/RedactionDlqMessageReceiver.java new file mode 100644 index 000000000..a1041a219 --- /dev/null +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/RedactionDlqMessageReceiver.java @@ -0,0 +1,52 @@ +package com.iqser.red.service.peristence.v1.server.service.download; + +import org.springframework.amqp.rabbit.annotation.RabbitHandler; +import org.springframework.amqp.rabbit.annotation.RabbitListener; +import org.springframework.stereotype.Service; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iqser.red.service.peristence.v1.server.configuration.MessagingConfiguration; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DownloadStatusPersistenceService; +import com.iqser.red.service.persistence.service.v1.api.model.download.DownloadStatusValue; + +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; +import lombok.experimental.FieldDefaults; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +@RequiredArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true) +@RabbitListener(queues = MessagingConfiguration.PDFTRON_DLQ) +public class RedactionDlqMessageReceiver { + + private static final String LINE_SEPARATOR = System.lineSeparator(); + + ObjectMapper objectMapper; + DownloadStatusPersistenceService downloadStatusPersistenceService; + + + @RabbitHandler + public void receive(String in) throws JsonProcessingException { + + // Since we receive different message types here, we do not convert to an object here; + // We just assume that the message contains a downloadId. + JsonNode jsonNode = objectMapper.readTree(in); + final String downloadId; + try { + downloadId = jsonNode.findValue("downloadId").asText(); + } catch (Exception e) { + log.warn("Received a message in the " + MessagingConfiguration.PDFTRON_DLQ + " that contains no downloadId" + LINE_SEPARATOR + "{}", jsonNode.asText()); + throw new RuntimeException(e); + } + + log.info("Received a dead message with downloadId:{}, updating the download as failed", downloadId); + + downloadStatusPersistenceService.updateStatus(downloadId, DownloadStatusValue.FAILED); + + } + +} diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/RedactionResultMessageReceiver.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/RedactionResultMessageReceiver.java new file mode 100644 index 000000000..246b8a72c --- /dev/null +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/RedactionResultMessageReceiver.java @@ -0,0 +1,38 @@ +package com.iqser.red.service.peristence.v1.server.service.download; + +import org.springframework.amqp.rabbit.annotation.RabbitHandler; +import org.springframework.amqp.rabbit.annotation.RabbitListener; +import org.springframework.stereotype.Service; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionResultMessage; +import com.iqser.red.service.peristence.v1.server.configuration.MessagingConfiguration; + +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; +import lombok.experimental.FieldDefaults; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +@RequiredArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true) +@RabbitListener(queues = MessagingConfiguration.PDFTRON_RESULT_QUEUE) +public class RedactionResultMessageReceiver { + + ObjectMapper objectMapper; + DownloadPreparationService downloadPreparationService; + + + @RabbitHandler + public void receive(String in) throws JsonProcessingException { + + RedactionResultMessage redactionResultMessage = objectMapper.readValue(in, RedactionResultMessage.class); + + log.info("Received redaction results for downloadId:{}", redactionResultMessage.getDownloadId()); + + downloadPreparationService.createDownload(redactionResultMessage); + } + +} diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/34-add-reports-information-column.changelog.yaml b/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/34-add-reports-information-column.changelog.yaml new file mode 100644 index 000000000..42032274e --- /dev/null +++ b/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/34-add-reports-information-column.changelog.yaml @@ -0,0 +1,11 @@ +databaseChangeLog: + - changeSet: + id: add-reports-information-column + author: viktorseifert + changes: + - addColumn: + columns: + - column: + name: generated_reports_information + type: VARCHAR(2000) + tableName: download_status diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/db.changelog-master.yaml b/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/db.changelog-master.yaml index 4f2e1fbe9..e2382618f 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/db.changelog-master.yaml +++ b/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/db.changelog-master.yaml @@ -81,3 +81,5 @@ databaseChangeLog: file: db/changelog/33-add-file-processing-error-counter-column.changelog.yaml - include: file: db/changelog/sql/33-set-file-processing-error-counter.sql + - include: + file: db/changelog/34-add-reports-information-column.changelog.yaml diff --git a/persistence-service-v1/pom.xml b/persistence-service-v1/pom.xml index d53903a98..58d00c915 100755 --- a/persistence-service-v1/pom.xml +++ b/persistence-service-v1/pom.xml @@ -27,7 +27,7 @@ 3.114.0 2.36.0 - 3.111.0 + 3.112.0 3.47.0