more tests and refactor downlod package to not use reportType

This commit is contained in:
Timo Bejan 2021-09-20 15:19:58 +03:00
parent 3a1662ef0f
commit 49ced1cb50
5 changed files with 197 additions and 105 deletions

View File

@ -1,23 +1,21 @@
package com.iqser.red.service.peristence.v1.server.service.download;
import com.iqser.red.service.pdftron.redaction.v1.api.model.PdfTronRedactionRequest;
import com.iqser.red.service.peristence.v1.server.client.RedactionClient;
import com.iqser.red.service.peristence.v1.server.service.FileManagementStorageService;
import com.iqser.red.service.peristence.v1.server.service.RedactionLogService;
import com.iqser.red.service.peristence.v1.server.utils.FileSystemBackedArchiver;
import com.iqser.red.service.pdftron.redaction.v1.api.model.PdfTronRedactionRequest;
import com.iqser.red.service.persistence.management.v1.processor.client.PDFTronRedactionClient;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DownloadStatusPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ReportTemplatePersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.*;
import com.iqser.red.service.persistence.service.v1.api.model.FileType;
import com.iqser.red.service.persistence.service.v1.api.model.data.audit.AddNotificationRequest;
import com.iqser.red.service.persistence.service.v1.api.model.data.dossier.Dossier;
import com.iqser.red.service.persistence.service.v1.api.model.data.dossier.DownloadFileType;
import com.iqser.red.service.persistence.service.v1.api.model.data.dossier.File;
import com.iqser.red.service.persistence.service.v1.api.model.data.dossier.ReportTemplate;
import com.iqser.red.service.persistence.service.v1.api.model.data.download.DownloadStatus;
import com.iqser.red.service.redaction.report.v1.api.model.ReportResultMessage;
import com.iqser.red.service.redaction.report.v1.api.model.ReportType;
import com.iqser.red.service.redaction.report.v1.api.model.StoredFileInformation;
import com.iqser.red.service.redaction.v1.model.AnnotateRequest;
import com.iqser.red.service.redaction.v1.model.RedactionLog;
@ -25,6 +23,10 @@ import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import javax.transaction.Transactional;
import java.util.Map;
import java.util.stream.Collectors;
@Slf4j
@Service
@RequiredArgsConstructor
@ -36,14 +38,15 @@ public class DownloadPreparationService {
private final RedactionClient redactionClient;
private final PDFTronRedactionClient pdfTronRedactionClient;
private final ReportTemplatePersistenceService reportTemplatePersistenceService;
private final DossierPersistenceService dossierPersistenceService;
private final RedactionLogService redactionLogService;
private final NotificationPersistenceService notificationPersistenceService;
private final DossierPersistenceService dossierPersistenceService;
@Transactional
public void createDownload(ReportResultMessage reportResultMessage) {
DownloadStatus downloadStatus = downloadStatusPersistenceService.getStatus(reportResultMessage.getDownloadId());
var dossier = downloadStatus.getDossier();
Dossier dossier = downloadStatus.getDossier();
try (FileSystemBackedArchiver fileSystemBackedArchiver = new FileSystemBackedArchiver()) {
@ -51,8 +54,14 @@ public class DownloadPreparationService {
addReports(reportResultMessage, fileSystemBackedArchiver);
storeZipFile(downloadStatus, fileSystemBackedArchiver);
downloadStatusPersistenceService.updateStatus(downloadStatus.getStorageId(), DownloadStatus.DownloadStatusValue.READY, fileSystemBackedArchiver
.getContentLength());
downloadStatusPersistenceService.updateStatus(downloadStatus.getStorageId(), DownloadStatus.DownloadStatusValue.READY, fileSystemBackedArchiver.getContentLength());
notificationPersistenceService.insertNotification(AddNotificationRequest.builder()
.userId(downloadStatus.getUserId())
.issuerId(downloadStatus.getUserId())
.notificationType("DOWNLOAD_READY")
.target(Map.of("userId", downloadStatus.getUserId(), "dossierId", dossier.getId(), "downloadId", reportResultMessage.getDownloadId()))
.build());
}
}
@ -60,18 +69,18 @@ public class DownloadPreparationService {
private void generateAndAddFiles(DownloadStatus downloadStatus, FileSystemBackedArchiver fileSystemBackedArchiver,
boolean watermarkEnabled) {
var dossier = downloadStatus.getDossier();
int i = 1;
long fileGenerationStart = System.currentTimeMillis();
for (File fileStatus : downloadStatus.getFiles()) {
var fileIds = downloadStatus.getFiles().stream().map(File::getId).collect(Collectors.toList());
for (String fileId : fileIds) {
var fileId = fileStatus.getId();
long start = System.currentTimeMillis();
byte[] original = fileManagementStorageService.getStoredObjectBytes(dossier.getId(), fileId, FileType.ORIGIN);
File fileStatus = fileStatusPersistenceService.getStatus(fileId);
byte[] original = fileManagementStorageService.getStoredObjectBytes(fileStatus.getDossierId(), fileId, FileType.ORIGIN);
RedactionLog redactionLog = getRedactionLog(dossier.getId(), fileId, fileStatus.isExcluded());
RedactionLog redactionLog = getRedactionLog(fileStatus.getDossierId(), fileId, fileStatus.isExcluded());
var dossier = dossierPersistenceService.findByDossierId(fileStatus.getDossierId());
for (DownloadFileType downloadFileType : downloadStatus.getDownloadFileTypes()) {
@ -79,23 +88,20 @@ public class DownloadPreparationService {
fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel("Original", fileStatus.getFilename(), original));
}
if (downloadFileType.name().equals(DownloadFileType.ANNOTATED.name())) {
fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel("Annotated", addSuffix(fileStatus
.getFilename(), "annotated"), getAnnotated(dossier.getDossierTemplateId(), dossier.getId(), fileId)));
fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel("Annotated", addSuffix(fileStatus.getFilename(), "annotated"), getAnnotated(dossier.getDossierTemplateId(), fileStatus.getDossierId(), fileId)));
}
if (downloadFileType.name().equals(DownloadFileType.PREVIEW.name())) {
fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel("Preview", addSuffix(fileStatus
.getFilename(), "highlighted"), getPreview(original, redactionLog, dossier.getDossierTemplateId())));
fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel("Preview", addSuffix(fileStatus.getFilename(), "highlighted"), getPreview(original, redactionLog, dossier.getDossierTemplateId())));
}
if (downloadFileType.name().equals(DownloadFileType.REDACTED.name())) {
fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel("Redacted", addSuffix(fileStatus
.getFilename(), "redacted"), getRedacted(original, redactionLog, dossier.getDossierTemplateId(), watermarkEnabled)));
fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel("Redacted", addSuffix(fileStatus.getFilename(), "redacted"), getRedacted(original, redactionLog, dossier.getDossierTemplateId(), watermarkEnabled)));
}
}
log.info("Successfully added file {}/{} for downloadId {}, took {}", i, downloadStatus.getFiles()
log.info("Successfully added file {}/{} for downloadId {}, took {}", i, fileIds
.size(), downloadStatus.getStorageId(), System.currentTimeMillis() - start);
i++;
}
log.info("Successfully added {} files for downloadId {}, took {}", i, downloadStatus.getStorageId(), System.currentTimeMillis() - fileGenerationStart);
log.info("Successfully added {} files for downloadId {}, took {}", fileIds, downloadStatus.getStorageId(), System.currentTimeMillis() - fileGenerationStart);
}
@ -104,58 +110,57 @@ public class DownloadPreparationService {
long addReportsStart = System.currentTimeMillis();
for (StoredFileInformation storedFileInformation : reportResultMessage.getStoredFileInformation()) {
if (storedFileInformation.getReportType().equals(ReportType.EXCEL_MULTI_FILE)) {
byte[] report = fileManagementStorageService.getStoredObjectBytes(storedFileInformation.getStorageId());
fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel("Excel Reports", "MultiFileReport.xlsx", report));
continue;
File fileStatus = null;
if (storedFileInformation.getFileId() != null) {
fileStatus = fileStatusPersistenceService.getStatus(storedFileInformation.getFileId());
}
if (storedFileInformation.getReportType().equals(ReportType.EXCEL_TEMPLATE_MULTI_FILE)) {
ReportTemplate reportTemplate = reportTemplatePersistenceService.find(storedFileInformation.getTemplateId());
byte[] report = fileManagementStorageService.getStoredObjectBytes(storedFileInformation.getStorageId());
fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(removeExtension(reportTemplate
.getFileName()) + " (multifile)", reportTemplate.getFileName(), report));
continue;
}
ReportTemplate reportTemplate = reportTemplatePersistenceService.find(storedFileInformation.getTemplateId());
byte[] report = fileManagementStorageService.getStoredObjectBytes(storedFileInformation.getStorageId());
fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(removeExtension(reportTemplate.getFileName()) + (reportTemplate.isMultiFileReport() ? " (multifile)" : ""),
createFileName(fileStatus, reportTemplate), report));
var fileStatus = fileStatusPersistenceService.getStatus(storedFileInformation.getFileId());
if (storedFileInformation.getReportType().equals(ReportType.EXCEL_SINGLE_FILE)) {
byte[] report = fileManagementStorageService.getStoredObjectBytes(storedFileInformation.getStorageId());
fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel("Excel Reports", toXlsxFilename(fileStatus
.getFilename()), report));
}
if (storedFileInformation.getReportType().equals(ReportType.WORD_SINGLE_FILE_APPENDIX_A1_TEMPLATE)) {
byte[] report = fileManagementStorageService.getStoredObjectBytes(storedFileInformation.getStorageId());
fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel("Justification Appendix A1", toDocxFilename(addSuffix(fileStatus
.getFilename(), "justification")), report));
}
if (storedFileInformation.getReportType().equals(ReportType.WORD_SINGLE_FILE_APPENDIX_A2_TEMPLATE)) {
byte[] report = fileManagementStorageService.getStoredObjectBytes(storedFileInformation.getStorageId());
fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel("Justification Appendix A2", toDocxFilename(addSuffix(fileStatus
.getFilename(), "justification")), report));
}
if (storedFileInformation.getReportType().equals(ReportType.WORD_SINGLE_FILE)) {
ReportTemplate reportTemplate = reportTemplatePersistenceService.find(storedFileInformation.getTemplateId());
byte[] report = fileManagementStorageService.getStoredObjectBytes(storedFileInformation.getStorageId());
fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(removeExtension(reportTemplate
.getFileName()), toDocxFilename(fileStatus.getFilename()), report));
}
if (storedFileInformation.getReportType().equals(ReportType.EXCEL_TEMPLATE_SINGLE_FILE)) {
ReportTemplate reportTemplate = reportTemplatePersistenceService.find(storedFileInformation.getTemplateId());
byte[] report = fileManagementStorageService.getStoredObjectBytes(storedFileInformation.getStorageId());
fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(removeExtension(reportTemplate
.getFileName()), toXlsxFilename(fileStatus.getFilename()), report));
}
}
log.info("Successfully added {} reports for downloadId {}, took {}", reportResultMessage.getStoredFileInformation()
.size(), reportResultMessage.getDownloadId(), System.currentTimeMillis() - addReportsStart);
}
private String createFileName(File fileStatus, ReportTemplate reportTemplate) {
if (fileStatus != null) {
return removeExtension(fileStatus.getFilename()) + getExtension(reportTemplate.getFileName());
} else {
return reportTemplate.getFileName();
}
}
private String getExtension(String fileName) {
var index = fileName.lastIndexOf(".");
if (index > 0) {
return fileName.substring(index);
} else {
return "";
}
}
private String removeExtension(String fileName) {
var index = fileName.lastIndexOf(".");
if (index > 0) {
return fileName.substring(index);
} else {
return fileName;
}
}
private String addSuffix(String filename, String suffix) {
var oldExtension = getExtension(filename);
return removeExtension(filename) + "_" + suffix + oldExtension;
}
private void storeZipFile(DownloadStatus downloadStatus, FileSystemBackedArchiver fileSystemBackedArchiver) {
@ -197,38 +202,6 @@ public class DownloadPreparationService {
}
private String toDocxFilename(String filename) {
if (filename.toLowerCase().endsWith(".pdf")) {
return filename.substring(0, filename.length() - 3) + "docx";
} else {
return filename + ".docx";
}
}
private String toXlsxFilename(String filename) {
if (filename.toLowerCase().endsWith(".pdf")) {
return filename.substring(0, filename.length() - 3) + "xlsx";
} else {
return filename + ".xlsx";
}
}
private String removeExtension(String filename) {
return filename.substring(0, filename.length() - 5);
}
private String addSuffix(String filename, String suffix) {
return filename.substring(0, filename.length() - 4) + "_" + suffix + ".pdf";
}
private RedactionLog getRedactionLog(String dossierId, String fileId, boolean isExcluded) {
if (isExcluded) {

View File

@ -4,6 +4,7 @@ import com.google.common.collect.Sets;
import com.iqser.red.service.peristence.v1.server.integration.client.DossierClient;
import com.iqser.red.service.persistence.service.v1.api.model.CreateOrUpdateDossierRequest;
import com.iqser.red.service.persistence.service.v1.api.model.data.dossier.Dossier;
import com.iqser.red.service.persistence.service.v1.api.model.data.dossier.DossierTemplate;
import com.iqser.red.service.persistence.service.v1.api.model.data.dossier.DownloadFileType;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
@ -16,14 +17,12 @@ import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
public class DossierTesterAndProvider {
@Autowired
private DossierTemplateTesterAndProvider dossierTemplateTesterAndProvider;
private DossierTemplateTesterAndProvider dossierTemplateTesterAndProvider;
@Autowired
private DossierClient dossierClient;
public Dossier provideTestDossier(){
var testTemplate = dossierTemplateTesterAndProvider.provideTestTemplate();
public Dossier provideTestDossier(DossierTemplate testTemplate) {
CreateOrUpdateDossierRequest cru = new CreateOrUpdateDossierRequest();
cru.setDownloadFileTypes(Sets.newHashSet(DownloadFileType.ORIGINAL));
@ -48,7 +47,12 @@ public class DossierTesterAndProvider {
return loadedDossier;
}
public Dossier provideTestDossier() {
var testTemplate = dossierTemplateTesterAndProvider.provideTestTemplate();
return provideTestDossier(testTemplate);
}
}

View File

@ -0,0 +1,102 @@
package com.iqser.red.service.peristence.v1.server.integration.tests;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.peristence.v1.server.integration.client.DownloadClient;
import com.iqser.red.service.peristence.v1.server.integration.client.ReportTemplateClient;
import com.iqser.red.service.peristence.v1.server.integration.service.DossierTemplateTesterAndProvider;
import com.iqser.red.service.peristence.v1.server.integration.service.DossierTesterAndProvider;
import com.iqser.red.service.peristence.v1.server.integration.service.FileTesterAndProvider;
import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest;
import com.iqser.red.service.peristence.v1.server.service.download.DownloadReportMessageReceiver;
import com.iqser.red.service.persistence.service.v1.api.model.DownloadRequest;
import com.iqser.red.service.persistence.service.v1.api.model.ReportTemplateUploadRequest;
import com.iqser.red.service.redaction.report.v1.api.model.ReportResultMessage;
import com.iqser.red.service.redaction.report.v1.api.model.StoredFileInformation;
import com.iqser.red.storage.commons.service.StorageService;
import lombok.SneakyThrows;
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
import java.io.ByteArrayInputStream;
import java.util.ArrayList;
import java.util.Collections;
import static org.assertj.core.api.Assertions.assertThat;
public class DownloadPreparationTest extends AbstractPersistenceServerServiceTest {
@Autowired
private DownloadReportMessageReceiver downloadReportMessageReceiver;
@Autowired
private StorageService storageService;
@Autowired
private FileTesterAndProvider fileTesterAndProvider;
@Autowired
private DossierTesterAndProvider dossierTesterAndProvider;
@Autowired
private ObjectMapper objectMapper;
@Autowired
private DownloadClient downloadClient;
@Autowired
private DossierTemplateTesterAndProvider dossierTemplateTesterAndProvider;
@Autowired
private ReportTemplateClient reportTemplateClient;
@Test
@SneakyThrows
public void testReceiveDownloadPackage() {
var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate();
var dossier = dossierTesterAndProvider.provideTestDossier();
var file = fileTesterAndProvider.testAndProvideFile(dossier);
reportTemplateClient.uploadTemplate(ReportTemplateUploadRequest.builder()
.activeByDefault(true)
.dossierTemplateId(dossierTemplate.getId())
.multiFileReport(true)
.fileName("test.docx")
.template(new byte[]{1, 2, 3, 4}).build());
var availableTemplates = reportTemplateClient.getAvailableReportTemplates(dossierTemplate.getId());
assertThat(availableTemplates).isNotEmpty();
downloadClient.prepareDownload(DownloadRequest.builder()
.userId("1")
.dossierId(dossier.getId())
.fileIds(Collections.singletonList(file.getId()))
.build());
var statuses = downloadClient.getDownloadStatus("1");
assertThat(statuses).isNotEmpty();
assertThat(statuses.iterator().next().getLastDownload()).isNull();
ArrayList<StoredFileInformation> sivList = new ArrayList<>();
var siv = new StoredFileInformation();
siv.setFileId(file.getId());
siv.setStorageId("XYZ");
siv.setTemplateId(availableTemplates.iterator().next().getTemplateId());
sivList.add(siv);
storageService.storeObject("XYZ", new ByteArrayInputStream(new byte[]{1, 2, 3, 4}));
ReportResultMessage reportResultMessage = new ReportResultMessage();
reportResultMessage.setUserId("1");
reportResultMessage.setDownloadId(statuses.iterator().next().getStorageId());
reportResultMessage.setStoredFileInformation(sivList);
downloadReportMessageReceiver.receive(objectMapper.writeValueAsString(reportResultMessage));
}
}

View File

@ -54,4 +54,5 @@ public class DownloadTest extends AbstractPersistenceServerServiceTest {
}
}

View File

@ -9,6 +9,7 @@ import org.springframework.core.io.InputStreamResource;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.util.HashMap;
import java.util.Map;
@ -32,6 +33,17 @@ public class FileSystemBackedStorageService extends StorageService {
}
@SneakyThrows
@Override
public void storeObject(String objectId, InputStream data) {
File tempFile = File.createTempFile("test", ".tmp");
IOUtils.copy(data, new FileOutputStream(tempFile));
dataMap.put(objectId, tempFile);
}
@SneakyThrows
@Override
public void storeObject(String objectId, byte[] data) {