Pull request #468: RED-1098
Merge in RED/persistence-service from RED-1098 to master * commit 'afc837163677aaa33fe1c8c25f7b52c39dd41a66': RED-1098: Added files generated by pdftron to the list of files that are deleted when the download archive has been created RED-1098: Updated pdftron-redaction-service version to include the newly created DTO's RED-1098: Implemented a receiver for dead messages from the pdftron-exchanges RED-1098: Corrected typo in log-message RED-1098: Corrected queue setting on pdftron-message-receiver RED-1098: Moved cleanup behavior to the correct location (after receiving the pdftron-results) RED-1098: Added the information about the generated reports to the db & implemented processing the response from pdf-tron. RED-1098: Implemented writing the requests to the pdftron-service to the respective queue instead of sending them via http RED-1098: Added configuration for the pdftron-redaction queue's
This commit is contained in:
Viktor seifert
commit
cca7543699
@ -100,5 +100,9 @@
|
||||
<artifactId>guava</artifactId>
|
||||
<scope>compile</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.iqser.red.service</groupId>
|
||||
<artifactId>redaction-report-service-api-v1</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</project>
|
||||
|
||||
@ -1,62 +1,81 @@
|
||||
package com.iqser.red.service.persistence.management.v1.processor.entity.download;
|
||||
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.utils.JSONDownloadFileTypeConverter;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DownloadFileType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.download.DownloadStatusValue;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
import org.hibernate.annotations.BatchSize;
|
||||
import org.hibernate.annotations.Fetch;
|
||||
import org.hibernate.annotations.FetchMode;
|
||||
|
||||
import javax.persistence.*;
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
import javax.persistence.Column;
|
||||
import javax.persistence.Convert;
|
||||
import javax.persistence.Entity;
|
||||
import javax.persistence.EnumType;
|
||||
import javax.persistence.Enumerated;
|
||||
import javax.persistence.FetchType;
|
||||
import javax.persistence.Id;
|
||||
import javax.persistence.ManyToMany;
|
||||
import javax.persistence.ManyToOne;
|
||||
import javax.persistence.Table;
|
||||
|
||||
import org.hibernate.annotations.Fetch;
|
||||
import org.hibernate.annotations.FetchMode;
|
||||
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.utils.JSONDownloadFileTypeConverter;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.utils.JSONStoredFileInformationConverter;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DownloadFileType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.download.DownloadStatusValue;
|
||||
import com.iqser.red.service.redaction.report.v1.api.model.StoredFileInformation;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.experimental.FieldDefaults;
|
||||
|
||||
@Data
|
||||
@Entity
|
||||
@Builder
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
@FieldDefaults(level = AccessLevel.PRIVATE)
|
||||
@Table(name = "download_status")
|
||||
public class DownloadStatusEntity {
|
||||
|
||||
@Id
|
||||
private String storageId;
|
||||
String storageId;
|
||||
@Column
|
||||
private String userId;
|
||||
String userId;
|
||||
@Column
|
||||
private String filename;
|
||||
String filename;
|
||||
@Column
|
||||
private String mimeType;
|
||||
String mimeType;
|
||||
@Column
|
||||
@Enumerated(EnumType.STRING)
|
||||
private DownloadStatusValue status;
|
||||
DownloadStatusValue status;
|
||||
@Column
|
||||
private OffsetDateTime creationDate;
|
||||
OffsetDateTime creationDate;
|
||||
@Column
|
||||
private OffsetDateTime lastDownload;
|
||||
OffsetDateTime lastDownload;
|
||||
@Column
|
||||
private long fileSize;
|
||||
long fileSize;
|
||||
|
||||
@ManyToOne(fetch = FetchType.LAZY)
|
||||
private DossierEntity dossier;
|
||||
DossierEntity dossier;
|
||||
|
||||
@ManyToMany
|
||||
@Fetch(FetchMode.SUBSELECT)
|
||||
private List<FileEntity> files = new ArrayList<>();
|
||||
List<FileEntity> files = new ArrayList<>();
|
||||
|
||||
@Builder.Default
|
||||
@Column(columnDefinition = "text", name = "download_file_types")
|
||||
@Convert(converter = JSONDownloadFileTypeConverter.class)
|
||||
private Set<DownloadFileType> downloadFileTypes = new HashSet<>();
|
||||
|
||||
Set<DownloadFileType> downloadFileTypes = new HashSet<>();
|
||||
|
||||
@Builder.Default
|
||||
@Column(columnDefinition = "text", name = "generated_reports_information")
|
||||
@Convert(converter = JSONStoredFileInformationConverter.class)
|
||||
Set<StoredFileInformation> generatedReportsInformation = new HashSet<>();
|
||||
}
|
||||
|
||||
@ -1,5 +1,15 @@
|
||||
package com.iqser.red.service.persistence.management.v1.processor.service.persistence;
|
||||
|
||||
import java.time.OffsetDateTime;
|
||||
import java.time.temporal.ChronoUnit;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
import javax.transaction.Transactional;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.download.DownloadStatusEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
|
||||
@ -7,16 +17,8 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DownloadFileType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.download.DownloadStatusValue;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import javax.transaction.Transactional;
|
||||
import java.time.OffsetDateTime;
|
||||
import java.time.temporal.ChronoUnit;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@ -76,6 +78,11 @@ public class DownloadStatusPersistenceService {
|
||||
}
|
||||
|
||||
|
||||
public void updateStatusEntity(DownloadStatusEntity downloadStatusEntity) {
|
||||
downloadStatusRepository.save(downloadStatusEntity);
|
||||
}
|
||||
|
||||
|
||||
public List<DownloadStatusEntity> getStatusesByUser(String userId) {
|
||||
|
||||
return downloadStatusRepository.findAllByUserId(userId);
|
||||
|
||||
@ -0,0 +1,42 @@
|
||||
package com.iqser.red.service.persistence.management.v1.processor.utils;
|
||||
|
||||
import java.util.HashSet;
|
||||
import java.util.Set;
|
||||
|
||||
import javax.persistence.AttributeConverter;
|
||||
import javax.persistence.Converter;
|
||||
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.iqser.red.service.redaction.report.v1.api.model.StoredFileInformation;
|
||||
|
||||
import lombok.SneakyThrows;
|
||||
|
||||
@Converter
|
||||
public class JSONStoredFileInformationConverter implements AttributeConverter<Set<StoredFileInformation>, String> {
|
||||
|
||||
private final ObjectMapper objectMapper = new ObjectMapper();
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
@Override
|
||||
public String convertToDatabaseColumn(Set<StoredFileInformation> dataSet) {
|
||||
|
||||
return objectMapper.writeValueAsString(dataSet);
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
@Override
|
||||
public Set<StoredFileInformation> convertToEntityAttribute(String data) {
|
||||
|
||||
if (data == null) {
|
||||
return new HashSet<>();
|
||||
}
|
||||
TypeReference<HashSet<StoredFileInformation>> typeRef = new TypeReference<>() {
|
||||
};
|
||||
return objectMapper.readValue(data, typeRef);
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@ -48,6 +48,10 @@ public class MessagingConfiguration {
|
||||
public static final String PDF_2_IMAGE_RESPONSE_QUEUE = "pdf2image_response_queue";
|
||||
public static final String PDF_2_IMAGE_DLQ = "pdf2image_dead_letter_queue";
|
||||
|
||||
public static final String PDFTRON_QUEUE = "pdftron_queue";
|
||||
public static final String PDFTRON_DLQ = "pdftron_dlq";
|
||||
public static final String PDFTRON_RESULT_QUEUE = "pdftron_result_queue";
|
||||
|
||||
|
||||
@Bean
|
||||
public Queue nerRequestQueue() {
|
||||
@ -59,10 +63,7 @@ public class MessagingConfiguration {
|
||||
@Bean
|
||||
public Queue nerResponseQueue() {
|
||||
|
||||
return QueueBuilder.durable(NER_SERVICE_RESPONSE_QUEUE)
|
||||
.withArgument("x-dead-letter-exchange", "")
|
||||
.withArgument("x-dead-letter-routing-key", NER_SERVICE_DLQ)
|
||||
.build();
|
||||
return QueueBuilder.durable(NER_SERVICE_RESPONSE_QUEUE).withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", NER_SERVICE_DLQ).build();
|
||||
}
|
||||
|
||||
|
||||
@ -76,18 +77,14 @@ public class MessagingConfiguration {
|
||||
@Bean
|
||||
public Queue imageRequestQueue() {
|
||||
|
||||
return QueueBuilder.durable(IMAGE_SERVICE_QUEUE)
|
||||
.withArgument("x-dead-letter-exchange", "")
|
||||
.withArgument("x-dead-letter-routing-key", IMAGE_SERVICE_DLQ)
|
||||
.build();
|
||||
return QueueBuilder.durable(IMAGE_SERVICE_QUEUE).withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", IMAGE_SERVICE_DLQ).build();
|
||||
}
|
||||
|
||||
|
||||
@Bean
|
||||
public Queue imageResponseQueue() {
|
||||
|
||||
return QueueBuilder.durable(IMAGE_SERVICE_RESPONSE_QUEUE)
|
||||
.withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", IMAGE_SERVICE_DLQ).build();
|
||||
return QueueBuilder.durable(IMAGE_SERVICE_RESPONSE_QUEUE).withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", IMAGE_SERVICE_DLQ).build();
|
||||
}
|
||||
|
||||
|
||||
@ -152,10 +149,7 @@ public class MessagingConfiguration {
|
||||
@Bean
|
||||
public Queue downloadQueue() {
|
||||
|
||||
return QueueBuilder.durable(DOWNLOAD_QUEUE)
|
||||
.withArgument("x-dead-letter-exchange", "")
|
||||
.withArgument("x-dead-letter-routing-key", DOWNLOAD_DLQ)
|
||||
.build();
|
||||
return QueueBuilder.durable(DOWNLOAD_QUEUE).withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", DOWNLOAD_DLQ).build();
|
||||
}
|
||||
|
||||
|
||||
@ -188,10 +182,7 @@ public class MessagingConfiguration {
|
||||
@Bean
|
||||
public Queue reportResultQueue() {
|
||||
|
||||
return QueueBuilder.durable(REPORT_RESULT_QUEUE)
|
||||
.withArgument("x-dead-letter-exchange", "")
|
||||
.withArgument("x-dead-letter-routing-key", REPORT_RESULT_DLQ)
|
||||
.build();
|
||||
return QueueBuilder.durable(REPORT_RESULT_QUEUE).withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", REPORT_RESULT_DLQ).build();
|
||||
}
|
||||
|
||||
|
||||
@ -205,11 +196,7 @@ public class MessagingConfiguration {
|
||||
@Bean
|
||||
public Queue indexingQueue() {
|
||||
|
||||
return QueueBuilder.durable(INDEXING_QUEUE)
|
||||
.withArgument("x-dead-letter-exchange", "")
|
||||
.withArgument("x-dead-letter-routing-key", INDEXING_DQL)
|
||||
.maxPriority(2)
|
||||
.build();
|
||||
return QueueBuilder.durable(INDEXING_QUEUE).withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", INDEXING_DQL).maxPriority(2).build();
|
||||
}
|
||||
|
||||
|
||||
@ -237,6 +224,7 @@ public class MessagingConfiguration {
|
||||
return QueueBuilder.durable(DELETE_FROM_INDEX_DLQ).build();
|
||||
}
|
||||
|
||||
|
||||
@Bean
|
||||
public Queue preprocessingQueue() {
|
||||
|
||||
@ -255,4 +243,35 @@ public class MessagingConfiguration {
|
||||
return QueueBuilder.durable(PRE_PROCESSING_DLQ).build();
|
||||
}
|
||||
|
||||
|
||||
@Bean
|
||||
public Queue pdfTronQueue() {
|
||||
|
||||
return QueueBuilder.durable(PDFTRON_QUEUE)
|
||||
.withArgument("x-dead-letter-exchange", "")
|
||||
.withArgument("x-dead-letter-routing-key", PDFTRON_DLQ)
|
||||
.withArgument("x-max-priority", 2)
|
||||
.maxPriority(2)
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
@Bean
|
||||
public Queue pdfTronDlq() {
|
||||
|
||||
return QueueBuilder.durable(PDFTRON_DLQ).build();
|
||||
}
|
||||
|
||||
|
||||
@Bean
|
||||
public Queue pdfTronResultQueue() {
|
||||
|
||||
return QueueBuilder.durable(PDFTRON_RESULT_QUEUE)
|
||||
.withArgument("x-dead-letter-exchange", "")
|
||||
.withArgument("x-dead-letter-routing-key", PDFTRON_DLQ)
|
||||
.withArgument("x-max-priority", 2)
|
||||
.maxPriority(2)
|
||||
.build();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -1,57 +1,121 @@
|
||||
package com.iqser.red.service.peristence.v1.server.service.download;
|
||||
|
||||
import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionRequest;
|
||||
import com.iqser.red.service.peristence.v1.server.client.RedactionClient;
|
||||
import java.io.IOException;
|
||||
import java.util.Collection;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import javax.transaction.Transactional;
|
||||
|
||||
import org.springframework.amqp.rabbit.core.RabbitTemplate;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionMessage;
|
||||
import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionResultDetail;
|
||||
import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionResultMessage;
|
||||
import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionType;
|
||||
import com.iqser.red.service.peristence.v1.server.configuration.MessagingConfiguration;
|
||||
import com.iqser.red.service.peristence.v1.server.service.FileManagementStorageService;
|
||||
import com.iqser.red.service.peristence.v1.server.service.RedactionLogService;
|
||||
import com.iqser.red.service.peristence.v1.server.utils.FileSystemBackedArchiver;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.client.PDFTronRedactionClient;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.ReportTemplateEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.download.DownloadStatusEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.*;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DownloadStatusPersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.NotificationPersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ReportTemplatePersistenceService;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.audit.AddNotificationRequest;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DownloadFileType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.download.DownloadStatusValue;
|
||||
import com.iqser.red.service.redaction.report.v1.api.model.ReportResultMessage;
|
||||
import com.iqser.red.service.redaction.report.v1.api.model.StoredFileInformation;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLog;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
import com.iqser.red.storage.commons.service.StorageService;
|
||||
|
||||
import javax.transaction.Transactional;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
import lombok.AccessLevel;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.experimental.FieldDefaults;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true)
|
||||
public class DownloadPreparationService {
|
||||
|
||||
private final DownloadStatusPersistenceService downloadStatusPersistenceService;
|
||||
private final FileStatusPersistenceService fileStatusPersistenceService;
|
||||
private final FileManagementStorageService fileManagementStorageService;
|
||||
private final RedactionClient redactionClient;
|
||||
private final PDFTronRedactionClient pdfTronRedactionClient;
|
||||
private final ReportTemplatePersistenceService reportTemplatePersistenceService;
|
||||
private final RedactionLogService redactionLogService;
|
||||
private final NotificationPersistenceService notificationPersistenceService;
|
||||
private final DossierPersistenceService dossierPersistenceService;
|
||||
DownloadStatusPersistenceService downloadStatusPersistenceService;
|
||||
FileStatusPersistenceService fileStatusPersistenceService;
|
||||
FileManagementStorageService fileManagementStorageService;
|
||||
ReportTemplatePersistenceService reportTemplatePersistenceService;
|
||||
NotificationPersistenceService notificationPersistenceService;
|
||||
RabbitTemplate rabbitTemplate;
|
||||
ObjectMapper objectMapper;
|
||||
StorageService storageService;
|
||||
|
||||
DownloadReportCleanupService downloadReportCleanupService;
|
||||
|
||||
|
||||
@Transactional
|
||||
public void createDownload(ReportResultMessage reportResultMessage) {
|
||||
public void createDownload(ReportResultMessage reportResultMessage) throws JsonProcessingException {
|
||||
|
||||
DownloadStatusEntity downloadStatus = downloadStatusPersistenceService.getStatus(reportResultMessage.getDownloadId());
|
||||
downloadStatus.setGeneratedReportsInformation(new HashSet<>(reportResultMessage.getStoredFileInformation()));
|
||||
downloadStatusPersistenceService.updateStatusEntity(downloadStatus);
|
||||
|
||||
DossierEntity dossier = downloadStatus.getDossier();
|
||||
|
||||
RedactionMessage message = RedactionMessage.builder()
|
||||
.dossierId(dossier.getId())
|
||||
.downloadId(reportResultMessage.getDownloadId())
|
||||
.redactionTypes(toPdfTronRedactionTypes(downloadStatus.getDownloadFileTypes()))
|
||||
.fileIds(downloadStatus.getFiles().stream().map(FileEntity::getId).collect(Collectors.toList()))
|
||||
.build();
|
||||
|
||||
log.info("Sending redaction request for downloadId:{} to pdftron-redaction-queue", message.getDownloadId());
|
||||
rabbitTemplate.convertAndSend(MessagingConfiguration.PDFTRON_QUEUE, objectMapper.writeValueAsString(message));
|
||||
}
|
||||
|
||||
|
||||
private List<RedactionType> toPdfTronRedactionTypes(Collection<DownloadFileType> downloadFileTypes) {
|
||||
|
||||
var result = new LinkedList<RedactionType>();
|
||||
for (var downloadFileType : downloadFileTypes) {
|
||||
switch (downloadFileType) {
|
||||
case REDACTED:
|
||||
result.add(RedactionType.REDACTED);
|
||||
break;
|
||||
case PREVIEW:
|
||||
result.add(RedactionType.PREVIEW);
|
||||
break;
|
||||
case DELTA_PREVIEW:
|
||||
result.add(RedactionType.DELTA);
|
||||
break;
|
||||
default:
|
||||
// Other types don't need to be passed to pdf-tron
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
@Transactional
|
||||
public void createDownload(RedactionResultMessage reportResultMessage) {
|
||||
|
||||
DownloadStatusEntity downloadStatus = downloadStatusPersistenceService.getStatus(reportResultMessage.getDownloadId());
|
||||
|
||||
try (FileSystemBackedArchiver fileSystemBackedArchiver = new FileSystemBackedArchiver()) {
|
||||
|
||||
generateAndAddFiles(downloadStatus, fileSystemBackedArchiver);
|
||||
addReports(reportResultMessage, fileSystemBackedArchiver);
|
||||
generateAndAddFiles(downloadStatus, reportResultMessage, fileSystemBackedArchiver);
|
||||
addReports(downloadStatus, fileSystemBackedArchiver);
|
||||
storeZipFile(downloadStatus, fileSystemBackedArchiver);
|
||||
|
||||
downloadStatusPersistenceService.updateStatus(downloadStatus.getStorageId(), DownloadStatusValue.READY, fileSystemBackedArchiver.getContentLength());
|
||||
@ -60,13 +124,24 @@ public class DownloadPreparationService {
|
||||
.userId(downloadStatus.getUserId())
|
||||
.issuerId(downloadStatus.getUserId())
|
||||
.notificationType("DOWNLOAD_READY")
|
||||
.target(Map.of("userId", downloadStatus.getUserId(), "dossierId", dossier.getId(), "downloadId", reportResultMessage.getDownloadId()))
|
||||
.target(Map.of("userId", downloadStatus.getUserId(), //
|
||||
"dossierId", downloadStatus.getDossier().getId(), //
|
||||
"downloadId", downloadStatus.getStatus()))
|
||||
.build());
|
||||
}
|
||||
|
||||
downloadReportCleanupService.deleteTmpReportFiles(downloadStatus.getGeneratedReportsInformation()
|
||||
.stream()
|
||||
.map(StoredFileInformation::getStorageId)
|
||||
.collect(Collectors.toSet()));
|
||||
downloadReportCleanupService.deleteTmpReportFiles(reportResultMessage.getRedactionResultDetails()
|
||||
.stream()
|
||||
.map(RedactionResultDetail::getStorageId)
|
||||
.collect(Collectors.toSet()));
|
||||
}
|
||||
|
||||
|
||||
private void generateAndAddFiles(DownloadStatusEntity downloadStatus, FileSystemBackedArchiver fileSystemBackedArchiver) {
|
||||
private void generateAndAddFiles(DownloadStatusEntity downloadStatus, RedactionResultMessage reportResultMessage, FileSystemBackedArchiver fileSystemBackedArchiver) {
|
||||
|
||||
int i = 1;
|
||||
long fileGenerationStart = System.currentTimeMillis();
|
||||
@ -77,42 +152,69 @@ public class DownloadPreparationService {
|
||||
FileEntity fileStatus = fileStatusPersistenceService.getStatus(fileId);
|
||||
byte[] original = fileManagementStorageService.getStoredObjectBytes(fileStatus.getDossierId(), fileId, FileType.ORIGIN);
|
||||
|
||||
RedactionLog redactionLog = getRedactionLog(fileStatus.getDossierId(), fileId, fileStatus.isExcluded());
|
||||
|
||||
var dossier = dossierPersistenceService.findByDossierId(fileStatus.getDossierId());
|
||||
|
||||
for (DownloadFileType downloadFileType : downloadStatus.getDownloadFileTypes()) {
|
||||
|
||||
if (downloadFileType.name().equals(DownloadFileType.ORIGINAL.name())) {
|
||||
fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel("Original", fileStatus.getFilename(), original));
|
||||
}
|
||||
if (downloadFileType.name().equals(DownloadFileType.PREVIEW.name())) {
|
||||
fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel("Preview", addSuffix(fileStatus.getFilename(), "highlighted"),
|
||||
getPreview(dossier.getId(), fileId, redactionLog, dossier.getDossierTemplateId(), dossier.getPreviewWatermarkId())));
|
||||
fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel("Preview", addSuffix(fileStatus.getFilename(), "highlighted"), //
|
||||
getPreview(fileId, reportResultMessage.getRedactionResultDetails())));
|
||||
}
|
||||
if (downloadFileType.name().equals(DownloadFileType.DELTA_PREVIEW.name())) {
|
||||
fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel("Delta Preview", addSuffix(fileStatus.getFilename(), "delta_highlighted"),
|
||||
getDeltaPreview(dossier.getId(), fileId, redactionLog, dossier.getDossierTemplateId(), dossier.getPreviewWatermarkId())));
|
||||
fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel("Delta Preview", addSuffix(fileStatus.getFilename(), "delta_highlighted"), //
|
||||
getDeltaPreview(fileId, reportResultMessage.getRedactionResultDetails())));
|
||||
}
|
||||
if (downloadFileType.name().equals(DownloadFileType.REDACTED.name())) {
|
||||
fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel("Redacted", addSuffix(fileStatus.getFilename(), "redacted"),
|
||||
getRedacted(dossier.getId(), fileId, redactionLog, dossier.getDossierTemplateId(), dossier.getWatermarkId())));
|
||||
fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel("Redacted", addSuffix(fileStatus.getFilename(), "redacted"), //
|
||||
getRedacted(fileId, reportResultMessage.getRedactionResultDetails())));
|
||||
}
|
||||
}
|
||||
log.info("Successfully added file {}/{} for downloadId {}, took {}", i, fileIds
|
||||
.size(), downloadStatus.getStorageId(), System.currentTimeMillis() - start);
|
||||
log.info("Successfully added file {}/{} for downloadId {}, took {}", i, fileIds.size(), downloadStatus.getStorageId(), System.currentTimeMillis() - start);
|
||||
i++;
|
||||
}
|
||||
log.info("Successfully added {} files for downloadId {}, took {}", fileIds, downloadStatus.getStorageId(), System.currentTimeMillis() - fileGenerationStart);
|
||||
}
|
||||
|
||||
|
||||
private void addReports(ReportResultMessage reportResultMessage, FileSystemBackedArchiver fileSystemBackedArchiver) {
|
||||
private byte[] getRedacted(String fileId, List<RedactionResultDetail> redactionResultDetails) {
|
||||
|
||||
return getStoredFileBytes(fileId, redactionResultDetails, RedactionType.REDACTED);
|
||||
}
|
||||
|
||||
|
||||
private byte[] getStoredFileBytes(String fileId, List<RedactionResultDetail> redactionResultDetails, RedactionType redactionType) {
|
||||
|
||||
var redactionResultDetail = redactionResultDetails.stream().filter(rrd -> Objects.equals(fileId, rrd.getFileId()) && rrd.getRedactionType() == redactionType).findFirst();
|
||||
if (redactionResultDetail.isPresent()) {
|
||||
try (var inputStream = storageService.getObject(redactionResultDetail.get().getStorageId()).getInputStream()) {
|
||||
return inputStream.readAllBytes();
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
throw new RuntimeException(String.format("Result for file %s of type %s not found", fileId, redactionType));
|
||||
}
|
||||
|
||||
|
||||
private byte[] getDeltaPreview(String fileId, List<RedactionResultDetail> redactionResultDetails) {
|
||||
|
||||
return getStoredFileBytes(fileId, redactionResultDetails, RedactionType.DELTA);
|
||||
}
|
||||
|
||||
|
||||
private byte[] getPreview(String fileId, List<RedactionResultDetail> redactionResultDetails) {
|
||||
|
||||
return getStoredFileBytes(fileId, redactionResultDetails, RedactionType.PREVIEW);
|
||||
}
|
||||
|
||||
|
||||
private void addReports(DownloadStatusEntity downloadStatus, FileSystemBackedArchiver fileSystemBackedArchiver) {
|
||||
|
||||
long addReportsStart = System.currentTimeMillis();
|
||||
|
||||
for (StoredFileInformation storedFileInformation : reportResultMessage.getStoredFileInformation()) {
|
||||
|
||||
for (StoredFileInformation storedFileInformation : downloadStatus.getGeneratedReportsInformation()) {
|
||||
|
||||
FileEntity fileStatus = null;
|
||||
|
||||
@ -122,14 +224,14 @@ public class DownloadPreparationService {
|
||||
|
||||
ReportTemplateEntity reportTemplate = reportTemplatePersistenceService.find(storedFileInformation.getTemplateId());
|
||||
byte[] report = fileManagementStorageService.getStoredObjectBytes(storedFileInformation.getStorageId());
|
||||
fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(removeExtension(reportTemplate.getFileName()) + (reportTemplate.isMultiFileReport() ? " (multifile)" : ""),
|
||||
addSuffix(createFileName(fileStatus, reportTemplate), "justification"), report));
|
||||
fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(removeExtension(reportTemplate.getFileName()) + (reportTemplate.isMultiFileReport() ? " (multifile)" : ""), addSuffix(createFileName(fileStatus, reportTemplate), "justification"), report));
|
||||
|
||||
}
|
||||
log.info("Successfully added {} reports for downloadId {}, took {}", reportResultMessage.getStoredFileInformation()
|
||||
.size(), reportResultMessage.getDownloadId(), System.currentTimeMillis() - addReportsStart);
|
||||
log.info("Successfully added {} reports for downloadId {}, took {}", downloadStatus.getGeneratedReportsInformation()
|
||||
.size(), downloadStatus.getStorageId(), System.currentTimeMillis() - addReportsStart);
|
||||
}
|
||||
|
||||
|
||||
private String createFileName(FileEntity fileStatus, ReportTemplateEntity reportTemplate) {
|
||||
|
||||
if (fileStatus != null) {
|
||||
@ -139,7 +241,9 @@ public class DownloadPreparationService {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private String getExtension(String fileName) {
|
||||
|
||||
var index = fileName.lastIndexOf(".");
|
||||
if (index > 0) {
|
||||
return fileName.substring(index);
|
||||
@ -148,7 +252,9 @@ public class DownloadPreparationService {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private String removeExtension(String fileName) {
|
||||
|
||||
var index = fileName.lastIndexOf(".");
|
||||
if (index > 0) {
|
||||
return fileName.substring(0, index);
|
||||
@ -159,6 +265,7 @@ public class DownloadPreparationService {
|
||||
|
||||
|
||||
private String addSuffix(String filename, String suffix) {
|
||||
|
||||
var oldExtension = getExtension(filename);
|
||||
return removeExtension(filename) + "_" + suffix + oldExtension;
|
||||
|
||||
@ -172,54 +279,4 @@ public class DownloadPreparationService {
|
||||
log.info("Successfully stored zip for downloadId {}, took {}", downloadStatus.getStorageId(), System.currentTimeMillis() - start);
|
||||
}
|
||||
|
||||
private byte[] getRedacted(String dossierId, String fileId, RedactionLog redactionLog, String dossierTemplateId,
|
||||
Long watermarkId) {
|
||||
|
||||
return pdfTronRedactionClient.redact(RedactionRequest.builder()
|
||||
.dossierId(dossierId)
|
||||
.fileId(fileId)
|
||||
.dossierTemplateId(dossierTemplateId)
|
||||
.redactionLog(redactionLog)
|
||||
.watermarkId(watermarkId)
|
||||
.build()).getDocument();
|
||||
}
|
||||
|
||||
|
||||
private byte[] getPreview(String dossierId, String fileId, RedactionLog redactionLog, String dossierTemplateId, Long watermarkId) {
|
||||
|
||||
return pdfTronRedactionClient.redactionPreview(RedactionRequest.builder()
|
||||
.dossierId(dossierId)
|
||||
.fileId(fileId)
|
||||
.dossierTemplateId(dossierTemplateId)
|
||||
.redactionLog(redactionLog)
|
||||
.watermarkId(watermarkId)
|
||||
.build()).getDocument();
|
||||
}
|
||||
|
||||
|
||||
private byte[] getDeltaPreview(String dossierId, String fileId, RedactionLog redactionLog, String dossierTemplateId, Long watermarkId) {
|
||||
|
||||
return pdfTronRedactionClient.redactionPreviewDiff(RedactionRequest.builder()
|
||||
.dossierId(dossierId)
|
||||
.fileId(fileId)
|
||||
.dossierTemplateId(dossierTemplateId)
|
||||
.redactionLog(redactionLog)
|
||||
.watermarkId(watermarkId)
|
||||
.build()).getDocument();
|
||||
}
|
||||
|
||||
|
||||
private RedactionLog getRedactionLog(String dossierId, String fileId, boolean isExcluded) {
|
||||
|
||||
if (isExcluded) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
return redactionLogService.getRedactionLog(dossierId, fileId, true, false);
|
||||
} catch (NotFoundException e) {
|
||||
return null;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -1,13 +1,15 @@
|
||||
package com.iqser.red.service.peristence.v1.server.service.download;
|
||||
|
||||
import com.iqser.red.service.peristence.v1.server.service.FileManagementStorageService;
|
||||
import com.iqser.red.service.redaction.report.v1.api.model.ReportResultMessage;
|
||||
import com.iqser.red.service.redaction.report.v1.api.model.StoredFileInformation;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import java.util.Collection;
|
||||
|
||||
import org.springframework.scheduling.annotation.Async;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.peristence.v1.server.service.FileManagementStorageService;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@ -16,11 +18,11 @@ public class DownloadReportCleanupService {
|
||||
private final FileManagementStorageService fileManagementStorageService;
|
||||
|
||||
@Async
|
||||
public void deleteTmpReportFiles(ReportResultMessage reportResultMessage) {
|
||||
public void deleteTmpReportFiles(Collection<String> storageIds) {
|
||||
|
||||
for (StoredFileInformation storedFileInformation : reportResultMessage.getStoredFileInformation()) {
|
||||
fileManagementStorageService.deleteObject(storedFileInformation.getStorageId());
|
||||
log.info("Deleted tmp report file {}", storedFileInformation.getStorageId());
|
||||
for (String storageId : storageIds) {
|
||||
fileManagementStorageService.deleteObject(storageId);
|
||||
log.info("Deleted tmp report file {}", storageId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,33 +1,38 @@
|
||||
package com.iqser.red.service.peristence.v1.server.service.download;
|
||||
|
||||
import org.springframework.amqp.rabbit.annotation.RabbitHandler;
|
||||
import org.springframework.amqp.rabbit.annotation.RabbitListener;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.iqser.red.service.peristence.v1.server.configuration.MessagingConfiguration;
|
||||
import com.iqser.red.service.redaction.report.v1.api.model.ReportResultMessage;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.experimental.FieldDefaults;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.amqp.rabbit.annotation.RabbitHandler;
|
||||
import org.springframework.amqp.rabbit.annotation.RabbitListener;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@RabbitListener(queues = MessagingConfiguration.REPORT_RESULT_QUEUE)
|
||||
@FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true)
|
||||
public class DownloadReportMessageReceiver {
|
||||
|
||||
private final ObjectMapper objectMapper;
|
||||
private final DownloadPreparationService downloadPreparationService;
|
||||
private final DownloadReportCleanupService downloadReportCleanupService;
|
||||
ObjectMapper objectMapper;
|
||||
DownloadPreparationService downloadPreparationService;
|
||||
|
||||
|
||||
@RabbitHandler
|
||||
public void receive(String in) throws JsonProcessingException {
|
||||
|
||||
ReportResultMessage reportResultMessage = objectMapper.readValue(in, ReportResultMessage.class);
|
||||
|
||||
log.info("Received request for userId:{} downloadId:{}", reportResultMessage.getUserId(), reportResultMessage.getDownloadId());
|
||||
|
||||
downloadPreparationService.createDownload(reportResultMessage);
|
||||
downloadReportCleanupService.deleteTmpReportFiles(reportResultMessage);
|
||||
log.info("Successfully prepared download {}", reportResultMessage.getDownloadId());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -0,0 +1,52 @@
|
||||
package com.iqser.red.service.peristence.v1.server.service.download;
|
||||
|
||||
import org.springframework.amqp.rabbit.annotation.RabbitHandler;
|
||||
import org.springframework.amqp.rabbit.annotation.RabbitListener;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.iqser.red.service.peristence.v1.server.configuration.MessagingConfiguration;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DownloadStatusPersistenceService;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.download.DownloadStatusValue;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.experimental.FieldDefaults;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true)
|
||||
@RabbitListener(queues = MessagingConfiguration.PDFTRON_DLQ)
|
||||
public class RedactionDlqMessageReceiver {
|
||||
|
||||
private static final String LINE_SEPARATOR = System.lineSeparator();
|
||||
|
||||
ObjectMapper objectMapper;
|
||||
DownloadStatusPersistenceService downloadStatusPersistenceService;
|
||||
|
||||
|
||||
@RabbitHandler
|
||||
public void receive(String in) throws JsonProcessingException {
|
||||
|
||||
// Since we receive different message types here, we do not convert to an object here;
|
||||
// We just assume that the message contains a downloadId.
|
||||
JsonNode jsonNode = objectMapper.readTree(in);
|
||||
final String downloadId;
|
||||
try {
|
||||
downloadId = jsonNode.findValue("downloadId").asText();
|
||||
} catch (Exception e) {
|
||||
log.warn("Received a message in the " + MessagingConfiguration.PDFTRON_DLQ + " that contains no downloadId" + LINE_SEPARATOR + "{}", jsonNode.asText());
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
|
||||
log.info("Received a dead message with downloadId:{}, updating the download as failed", downloadId);
|
||||
|
||||
downloadStatusPersistenceService.updateStatus(downloadId, DownloadStatusValue.FAILED);
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,38 @@
|
||||
package com.iqser.red.service.peristence.v1.server.service.download;
|
||||
|
||||
import org.springframework.amqp.rabbit.annotation.RabbitHandler;
|
||||
import org.springframework.amqp.rabbit.annotation.RabbitListener;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionResultMessage;
|
||||
import com.iqser.red.service.peristence.v1.server.configuration.MessagingConfiguration;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.experimental.FieldDefaults;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true)
|
||||
@RabbitListener(queues = MessagingConfiguration.PDFTRON_RESULT_QUEUE)
|
||||
public class RedactionResultMessageReceiver {
|
||||
|
||||
ObjectMapper objectMapper;
|
||||
DownloadPreparationService downloadPreparationService;
|
||||
|
||||
|
||||
@RabbitHandler
|
||||
public void receive(String in) throws JsonProcessingException {
|
||||
|
||||
RedactionResultMessage redactionResultMessage = objectMapper.readValue(in, RedactionResultMessage.class);
|
||||
|
||||
log.info("Received redaction results for downloadId:{}", redactionResultMessage.getDownloadId());
|
||||
|
||||
downloadPreparationService.createDownload(redactionResultMessage);
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,11 @@
|
||||
databaseChangeLog:
|
||||
- changeSet:
|
||||
id: add-reports-information-column
|
||||
author: viktorseifert
|
||||
changes:
|
||||
- addColumn:
|
||||
columns:
|
||||
- column:
|
||||
name: generated_reports_information
|
||||
type: VARCHAR(2000)
|
||||
tableName: download_status
|
||||
@ -81,3 +81,5 @@ databaseChangeLog:
|
||||
file: db/changelog/33-add-file-processing-error-counter-column.changelog.yaml
|
||||
- include:
|
||||
file: db/changelog/sql/33-set-file-processing-error-counter.sql
|
||||
- include:
|
||||
file: db/changelog/34-add-reports-information-column.changelog.yaml
|
||||
|
||||
@ -27,7 +27,7 @@
|
||||
<properties>
|
||||
<redaction-service.version>3.114.0</redaction-service.version>
|
||||
<search-service.version>2.36.0</search-service.version>
|
||||
<pdftron-redaction-service.version>3.111.0</pdftron-redaction-service.version>
|
||||
<pdftron-redaction-service.version>3.112.0</pdftron-redaction-service.version>
|
||||
<redaction-report-service.version>3.47.0</redaction-report-service.version>
|
||||
</properties>
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user