RED-1098: Added the information about the generated reports to the db & implemented processing the response from pdf-tron.

* Added the information about the generated reports to the download-entity so that it is available when the result from pdf-tron is received.
* Implemented processing the response from pdf-tron to complete the download archive, using the pdf-tron results from the queue instead of making rest-calls.
This commit is contained in:
Viktor Seifert 2022-06-28 17:23:28 +02:00
parent b8bf13d693
commit 4911fbff9c
8 changed files with 245 additions and 137 deletions

View File

@ -100,5 +100,9 @@
<artifactId>guava</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>com.iqser.red.service</groupId>
<artifactId>redaction-report-service-api-v1</artifactId>
</dependency>
</dependencies>
</project>

View File

@ -1,62 +1,81 @@
package com.iqser.red.service.persistence.management.v1.processor.entity.download;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
import com.iqser.red.service.persistence.management.v1.processor.utils.JSONDownloadFileTypeConverter;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DownloadFileType;
import com.iqser.red.service.persistence.service.v1.api.model.download.DownloadStatusValue;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.hibernate.annotations.BatchSize;
import org.hibernate.annotations.Fetch;
import org.hibernate.annotations.FetchMode;
import javax.persistence.*;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import javax.persistence.Column;
import javax.persistence.Convert;
import javax.persistence.Entity;
import javax.persistence.EnumType;
import javax.persistence.Enumerated;
import javax.persistence.FetchType;
import javax.persistence.Id;
import javax.persistence.ManyToMany;
import javax.persistence.ManyToOne;
import javax.persistence.Table;
import org.hibernate.annotations.Fetch;
import org.hibernate.annotations.FetchMode;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
import com.iqser.red.service.persistence.management.v1.processor.utils.JSONDownloadFileTypeConverter;
import com.iqser.red.service.persistence.management.v1.processor.utils.JSONStoredFileInformationConverter;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DownloadFileType;
import com.iqser.red.service.persistence.service.v1.api.model.download.DownloadStatusValue;
import com.iqser.red.service.redaction.report.v1.api.model.StoredFileInformation;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.experimental.FieldDefaults;
@Data
@Entity
@Builder
@AllArgsConstructor
@NoArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE)
@Table(name = "download_status")
public class DownloadStatusEntity {
@Id
private String storageId;
String storageId;
@Column
private String userId;
String userId;
@Column
private String filename;
String filename;
@Column
private String mimeType;
String mimeType;
@Column
@Enumerated(EnumType.STRING)
private DownloadStatusValue status;
DownloadStatusValue status;
@Column
private OffsetDateTime creationDate;
OffsetDateTime creationDate;
@Column
private OffsetDateTime lastDownload;
OffsetDateTime lastDownload;
@Column
private long fileSize;
long fileSize;
@ManyToOne(fetch = FetchType.LAZY)
private DossierEntity dossier;
DossierEntity dossier;
@ManyToMany
@Fetch(FetchMode.SUBSELECT)
private List<FileEntity> files = new ArrayList<>();
List<FileEntity> files = new ArrayList<>();
@Builder.Default
@Column(columnDefinition = "text", name = "download_file_types")
@Convert(converter = JSONDownloadFileTypeConverter.class)
private Set<DownloadFileType> downloadFileTypes = new HashSet<>();
Set<DownloadFileType> downloadFileTypes = new HashSet<>();
@Builder.Default
@Column(columnDefinition = "text", name = "generated_reports_information")
@Convert(converter = JSONStoredFileInformationConverter.class)
Set<StoredFileInformation> generatedReportsInformation = new HashSet<>();
}

View File

@ -1,5 +1,15 @@
package com.iqser.red.service.persistence.management.v1.processor.service.persistence;
import java.time.OffsetDateTime;
import java.time.temporal.ChronoUnit;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import javax.transaction.Transactional;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.download.DownloadStatusEntity;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
@ -7,16 +17,8 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DownloadFileType;
import com.iqser.red.service.persistence.service.v1.api.model.download.DownloadStatusValue;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
import javax.transaction.Transactional;
import java.time.OffsetDateTime;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import lombok.RequiredArgsConstructor;
@Service
@RequiredArgsConstructor
@ -76,6 +78,11 @@ public class DownloadStatusPersistenceService {
}
public void updateStatusEntity(DownloadStatusEntity downloadStatusEntity) {
downloadStatusRepository.save(downloadStatusEntity);
}
public List<DownloadStatusEntity> getStatusesByUser(String userId) {
return downloadStatusRepository.findAllByUserId(userId);

View File

@ -0,0 +1,42 @@
package com.iqser.red.service.persistence.management.v1.processor.utils;
import java.util.HashSet;
import java.util.Set;
import javax.persistence.AttributeConverter;
import javax.persistence.Converter;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.redaction.report.v1.api.model.StoredFileInformation;
import lombok.SneakyThrows;
@Converter
public class JSONStoredFileInformationConverter implements AttributeConverter<Set<StoredFileInformation>, String> {
private final ObjectMapper objectMapper = new ObjectMapper();
@SneakyThrows
@Override
public String convertToDatabaseColumn(Set<StoredFileInformation> dataSet) {
return objectMapper.writeValueAsString(dataSet);
}
@SneakyThrows
@Override
public Set<StoredFileInformation> convertToEntityAttribute(String data) {
if (data == null) {
return new HashSet<>();
}
TypeReference<HashSet<StoredFileInformation>> typeRef = new TypeReference<>() {
};
return objectMapper.readValue(data, typeRef);
}
}

View File

@ -1,6 +1,11 @@
package com.iqser.red.service.peristence.v1.server.service.download;
import java.io.IOException;
import java.util.Collection;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.stream.Collectors;
import javax.transaction.Transactional;
@ -11,20 +16,16 @@ import org.springframework.stereotype.Service;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionMessage;
import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionRequest;
import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionResultDetail;
import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionResultMessage;
import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionType;
import com.iqser.red.service.peristence.v1.server.client.RedactionClient;
import com.iqser.red.service.peristence.v1.server.configuration.MessagingConfiguration;
import com.iqser.red.service.peristence.v1.server.service.FileManagementStorageService;
import com.iqser.red.service.peristence.v1.server.service.RedactionLogService;
import com.iqser.red.service.peristence.v1.server.utils.FileSystemBackedArchiver;
import com.iqser.red.service.persistence.management.v1.processor.client.PDFTronRedactionClient;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.ReportTemplateEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.download.DownloadStatusEntity;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DownloadStatusPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.NotificationPersistenceService;
@ -35,7 +36,7 @@ import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.do
import com.iqser.red.service.persistence.service.v1.api.model.download.DownloadStatusValue;
import com.iqser.red.service.redaction.report.v1.api.model.ReportResultMessage;
import com.iqser.red.service.redaction.report.v1.api.model.StoredFileInformation;
import com.iqser.red.service.redaction.v1.model.RedactionLog;
import com.iqser.red.storage.commons.service.StorageService;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
@ -51,63 +52,66 @@ public class DownloadPreparationService {
DownloadStatusPersistenceService downloadStatusPersistenceService;
FileStatusPersistenceService fileStatusPersistenceService;
FileManagementStorageService fileManagementStorageService;
RedactionClient redactionClient;
PDFTronRedactionClient pdfTronRedactionClient;
ReportTemplatePersistenceService reportTemplatePersistenceService;
RedactionLogService redactionLogService;
NotificationPersistenceService notificationPersistenceService;
DossierPersistenceService dossierPersistenceService;
RabbitTemplate rabbitTemplate;
ObjectMapper objectMapper;
StorageService storageService;
@Transactional
public void createDownload(ReportResultMessage reportResultMessage) throws JsonProcessingException {
DownloadStatusEntity downloadStatus = downloadStatusPersistenceService.getStatus(reportResultMessage.getDownloadId());
downloadStatus.setGeneratedReportsInformation(downloadStatus.getGeneratedReportsInformation());
downloadStatusPersistenceService.updateStatusEntity(downloadStatus);
DossierEntity dossier = downloadStatus.getDossier();
RedactionMessage message = RedactionMessage.builder().build();
for (String fileId : downloadStatus.getFiles().stream().map(FileEntity::getId).collect(Collectors.toUnmodifiableList())) {
for (DownloadFileType downloadFileType : downloadStatus.getDownloadFileTypes()) {
RedactionType redactionType = toRedactionType(downloadFileType);
if (redactionType == null) {
continue;
}
message.getRedactionTypes().add(redactionType);
message.getFileIds().add(fileId);
message.getDossierIds().add(dossier.getId());
}
}
RedactionMessage message = RedactionMessage.builder()
.dossierId(dossier.getId())
.downloadId(reportResultMessage.getDownloadId())
.redactionTypes(toPdfTronRedactionTypes(downloadStatus.getDownloadFileTypes()))
.fileIds(downloadStatus.getFiles().stream().map(FileEntity::getId).collect(Collectors.toList()))
.build();
rabbitTemplate.convertAndSend(MessagingConfiguration.PDFTRON_QUEUE, objectMapper.writeValueAsString(message));
}
private RedactionType toRedactionType(DownloadFileType downloadFileType) {
private List<RedactionType> toPdfTronRedactionTypes(Collection<DownloadFileType> downloadFileTypes) {
switch (downloadFileType) {
case REDACTED:
return RedactionType.REDACTED;
case PREVIEW:
return RedactionType.PREVIEW;
case DELTA_PREVIEW:
return RedactionType.DELTA;
case ORIGINAL:
default:
return null;
var result = new LinkedList<RedactionType>();
for (var downloadFileType : downloadFileTypes) {
switch (downloadFileType) {
case REDACTED:
result.add(RedactionType.REDACTED);
break;
case PREVIEW:
result.add(RedactionType.PREVIEW);
break;
case DELTA_PREVIEW:
result.add(RedactionType.DELTA);
break;
default:
// Other types don't need to be passed to pdf-tron
break;
}
}
return result;
}
private void createDownloadArchive(ReportResultMessage reportResultMessage, DownloadStatusEntity downloadStatus, DossierEntity dossier) {
@Transactional
public void createDownload(RedactionResultMessage reportResultMessage) {
DownloadStatusEntity downloadStatus = downloadStatusPersistenceService.getStatus(reportResultMessage.getDownloadId());
try (FileSystemBackedArchiver fileSystemBackedArchiver = new FileSystemBackedArchiver()) {
generateAndAddFiles(downloadStatus, fileSystemBackedArchiver);
addReports(reportResultMessage, fileSystemBackedArchiver);
generateAndAddFiles(downloadStatus, reportResultMessage, fileSystemBackedArchiver);
addReports(downloadStatus, fileSystemBackedArchiver);
storeZipFile(downloadStatus, fileSystemBackedArchiver);
downloadStatusPersistenceService.updateStatus(downloadStatus.getStorageId(), DownloadStatusValue.READY, fileSystemBackedArchiver.getContentLength());
@ -116,13 +120,15 @@ public class DownloadPreparationService {
.userId(downloadStatus.getUserId())
.issuerId(downloadStatus.getUserId())
.notificationType("DOWNLOAD_READY")
.target(Map.of("userId", downloadStatus.getUserId(), "dossierId", dossier.getId(), "downloadId", reportResultMessage.getDownloadId()))
.target(Map.of("userId", downloadStatus.getUserId(), //
"dossierId", downloadStatus.getDossier().getId(), //
"downloadId", downloadStatus.getStatus()))
.build());
}
}
private void generateAndAddFiles(DownloadStatusEntity downloadStatus, FileSystemBackedArchiver fileSystemBackedArchiver) {
private void generateAndAddFiles(DownloadStatusEntity downloadStatus, RedactionResultMessage reportResultMessage, FileSystemBackedArchiver fileSystemBackedArchiver) {
int i = 1;
long fileGenerationStart = System.currentTimeMillis();
@ -133,23 +139,22 @@ public class DownloadPreparationService {
FileEntity fileStatus = fileStatusPersistenceService.getStatus(fileId);
byte[] original = fileManagementStorageService.getStoredObjectBytes(fileStatus.getDossierId(), fileId, FileType.ORIGIN);
RedactionLog redactionLog = getRedactionLog(fileStatus.getDossierId(), fileId, fileStatus.isExcluded());
var dossier = dossierPersistenceService.findByDossierId(fileStatus.getDossierId());
for (DownloadFileType downloadFileType : downloadStatus.getDownloadFileTypes()) {
if (downloadFileType.name().equals(DownloadFileType.ORIGINAL.name())) {
fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel("Original", fileStatus.getFilename(), original));
}
if (downloadFileType.name().equals(DownloadFileType.PREVIEW.name())) {
fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel("Preview", addSuffix(fileStatus.getFilename(), "highlighted"), getPreview(dossier.getId(), fileId, redactionLog, dossier.getDossierTemplateId(), dossier.getPreviewWatermarkId())));
fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel("Preview", addSuffix(fileStatus.getFilename(), "highlighted"), //
getPreview(fileId, reportResultMessage.getRedactionResultDetails())));
}
if (downloadFileType.name().equals(DownloadFileType.DELTA_PREVIEW.name())) {
fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel("Delta Preview", addSuffix(fileStatus.getFilename(), "delta_highlighted"), getDeltaPreview(dossier.getId(), fileId, redactionLog, dossier.getDossierTemplateId(), dossier.getPreviewWatermarkId())));
fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel("Delta Preview", addSuffix(fileStatus.getFilename(), "delta_highlighted"), //
getDeltaPreview(fileId, reportResultMessage.getRedactionResultDetails())));
}
if (downloadFileType.name().equals(DownloadFileType.REDACTED.name())) {
fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel("Redacted", addSuffix(fileStatus.getFilename(), "redacted"), getRedacted(dossier.getId(), fileId, redactionLog, dossier.getDossierTemplateId(), dossier.getWatermarkId())));
fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel("Redacted", addSuffix(fileStatus.getFilename(), "redacted"), //
getRedacted(fileId, reportResultMessage.getRedactionResultDetails())));
}
}
log.info("Successfully added file {}/{} for downloadId {}, took {}", i, fileIds.size(), downloadStatus.getStorageId(), System.currentTimeMillis() - start);
@ -159,11 +164,46 @@ public class DownloadPreparationService {
}
private void addReports(ReportResultMessage reportResultMessage, FileSystemBackedArchiver fileSystemBackedArchiver) {
private byte[] getRedacted(String fileId, List<RedactionResultDetail> redactionResultDetails) {
return getStoredFileBytes(fileId, redactionResultDetails, RedactionType.REDACTED);
}
private byte[] getStoredFileBytes(String fileId, List<RedactionResultDetail> redactionResultDetails, RedactionType redactionType) {
var redactionResultDetail = redactionResultDetails.stream()
.filter(rrd -> Objects.equals(fileId, rrd.getFileId()) && rrd.getRedactionType() == redactionType)
.findFirst();
if (redactionResultDetail.isPresent()) {
try (var inputStream = storageService.getObject(redactionResultDetail.get().getStorageId()).getInputStream()) {
return inputStream.readAllBytes();
} catch (IOException e) {
throw new RuntimeException(e);
}
}
throw new RuntimeException(String.format("Result for file %s of type %s not found", fileId, redactionType));
}
private byte[] getDeltaPreview(String fileId, List<RedactionResultDetail> redactionResultDetails) {
return getStoredFileBytes(fileId, redactionResultDetails, RedactionType.DELTA);
}
private byte[] getPreview(String fileId, List<RedactionResultDetail> redactionResultDetails) {
return getStoredFileBytes(fileId, redactionResultDetails, RedactionType.PREVIEW);
}
private void addReports(DownloadStatusEntity downloadStatus, FileSystemBackedArchiver fileSystemBackedArchiver) {
long addReportsStart = System.currentTimeMillis();
for (StoredFileInformation storedFileInformation : reportResultMessage.getStoredFileInformation()) {
for (StoredFileInformation storedFileInformation : downloadStatus.getGeneratedReportsInformation()) {
FileEntity fileStatus = null;
@ -176,8 +216,8 @@ public class DownloadPreparationService {
fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(removeExtension(reportTemplate.getFileName()) + (reportTemplate.isMultiFileReport() ? " (multifile)" : ""), addSuffix(createFileName(fileStatus, reportTemplate), "justification"), report));
}
log.info("Successfully added {} reports for downloadId {}, took {}", reportResultMessage.getStoredFileInformation()
.size(), reportResultMessage.getDownloadId(), System.currentTimeMillis() - addReportsStart);
log.info("Successfully added {} reports for downloadId {}, took {}", downloadStatus.getGeneratedReportsInformation()
.size(), downloadStatus.getStorageId(), System.currentTimeMillis() - addReportsStart);
}
@ -228,54 +268,4 @@ public class DownloadPreparationService {
log.info("Successfully stored zip for downloadId {}, took {}", downloadStatus.getStorageId(), System.currentTimeMillis() - start);
}
private byte[] getRedacted(String dossierId, String fileId, RedactionLog redactionLog, String dossierTemplateId, Long watermarkId) {
return pdfTronRedactionClient.redact(RedactionRequest.builder()
.dossierId(dossierId)
.fileId(fileId)
.dossierTemplateId(dossierTemplateId)
.redactionLog(redactionLog)
.watermarkId(watermarkId)
.build()).getDocument();
}
private byte[] getPreview(String dossierId, String fileId, RedactionLog redactionLog, String dossierTemplateId, Long watermarkId) {
return pdfTronRedactionClient.redactionPreview(RedactionRequest.builder()
.dossierId(dossierId)
.fileId(fileId)
.dossierTemplateId(dossierTemplateId)
.redactionLog(redactionLog)
.watermarkId(watermarkId)
.build()).getDocument();
}
private byte[] getDeltaPreview(String dossierId, String fileId, RedactionLog redactionLog, String dossierTemplateId, Long watermarkId) {
return pdfTronRedactionClient.redactionPreviewDiff(RedactionRequest.builder()
.dossierId(dossierId)
.fileId(fileId)
.dossierTemplateId(dossierTemplateId)
.redactionLog(redactionLog)
.watermarkId(watermarkId)
.build()).getDocument();
}
private RedactionLog getRedactionLog(String dossierId, String fileId, boolean isExcluded) {
if (isExcluded) {
return null;
}
try {
return redactionLogService.getRedactionLog(dossierId, fileId, true, false);
} catch (NotFoundException e) {
return null;
}
}
}

View File

@ -0,0 +1,33 @@
package com.iqser.red.service.peristence.v1.server.service.download;
import org.springframework.amqp.rabbit.annotation.RabbitHandler;
import org.springframework.amqp.rabbit.annotation.RabbitListener;
import org.springframework.stereotype.Service;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionResultMessage;
import com.iqser.red.service.peristence.v1.server.configuration.MessagingConfiguration;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.experimental.FieldDefaults;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@RequiredArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true)
@RabbitListener(queues = MessagingConfiguration.REPORT_RESULT_QUEUE)
public class RedactionResultMessageReceiver {
ObjectMapper objectMapper;
DownloadPreparationService downloadPreparationService;
@RabbitHandler
public void receive(String in) throws JsonProcessingException {
RedactionResultMessage redactionResultMessage = objectMapper.readValue(in, RedactionResultMessage.class);
downloadPreparationService.createDownload(redactionResultMessage);
}
}

View File

@ -0,0 +1,11 @@
databaseChangeLog:
- changeSet:
id: add-reports-information-column
author: viktorseifert
changes:
- addColumn:
columns:
- column:
name: generated_reports_information
type: VARCHAR(2000)
tableName: download_status

View File

@ -81,3 +81,5 @@ databaseChangeLog:
file: db/changelog/33-add-file-processing-error-counter-column.changelog.yaml
- include:
file: db/changelog/sql/33-set-file-processing-error-counter.sql
- include:
file: db/changelog/34-add-reports-information-column.changelog.yaml