Marmelator/redaction-report-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 129 Wiki Activity

RED-5047 - concurent report generation

Browse Source
This commit is contained in:
Timo Bejan 2022-09-06 11:08:23 +03:00
parent 6043f2af85
commit 62ccb3b827
5 changed files with 94 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/model/MultiFileWorkbook.java
View File

@ -14,5 +14,6 @@ public class MultiFileWorkbook {
private SXSSFWorkbook writeWorkbook;
private String templateId;
private String templateName;
private ExcelModel excelModel;
}

41
redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ReportGenerationService.java
View File

@ -1,19 +1,5 @@
package com.iqser.red.service.redaction.report.v1.server.service;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.apache.poi.xwpf.usermodel.XWPFDocument;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.ReportTemplate;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.Dossier;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel;
@ -21,12 +7,7 @@ import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.ty
import com.iqser.red.service.redaction.report.v1.api.model.ReportRequestMessage;
import com.iqser.red.service.redaction.report.v1.api.model.ReportType;
import com.iqser.red.service.redaction.report.v1.api.model.StoredFileInformation;
import com.iqser.red.service.redaction.report.v1.server.client.DictionaryClient;
import com.iqser.red.service.redaction.report.v1.server.client.DossierClient;
import com.iqser.red.service.redaction.report.v1.server.client.FileStatusClient;
import com.iqser.red.service.redaction.report.v1.server.client.RedactionLogClient;
import com.iqser.red.service.redaction.report.v1.server.client.ReportTemplateClient;
import com.iqser.red.service.redaction.report.v1.server.model.ExcelModel;
import com.iqser.red.service.redaction.report.v1.server.client.*;
import com.iqser.red.service.redaction.report.v1.server.model.MultiFileDocument;
import com.iqser.red.service.redaction.report.v1.server.model.MultiFileWorkbook;
import com.iqser.red.service.redaction.report.v1.server.model.PlaceholderModel;
@ -35,11 +16,19 @@ import com.iqser.red.service.redaction.report.v1.server.settings.ReportTemplateS
import com.iqser.red.service.redaction.report.v1.server.storage.ReportStorageService;
import com.iqser.red.service.redaction.v1.model.RedactionLog;
import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
import io.micrometer.core.annotation.Timed;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.apache.poi.xwpf.usermodel.XWPFDocument;
import org.springframework.stereotype.Service;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.util.*;
@SuppressWarnings("ALL")
@Slf4j
@ -82,7 +71,8 @@ public class ReportGenerationService {
for (Sheet sheet : readWorkbook) {
writeWorkbook.createSheet(sheet.getSheetName());
}
MultiFileWorkbook multiFileWorkbook = new MultiFileWorkbook(readWorkbook, writeWorkbook, templateId, reportTemplate.getFileName());
MultiFileWorkbook multiFileWorkbook = new MultiFileWorkbook(readWorkbook, writeWorkbook, templateId, reportTemplate.getFileName(),
excelTemplateReportGenerationService.calculateExcelModel(readWorkbook.getSheetAt(0)));
multiFileWorkbooks.add(multiFileWorkbook);
} catch (IOException e) {
throw new RuntimeException("Could not generate multifile excel report.");
@ -107,7 +97,6 @@ public class ReportGenerationService {
var placeholderModel = generatePlaceholderService.buildPlaceholders(dossier);
ExcelModel excelModel = null;
int i = 1;
for (int j = 0; j < reportMessage.getFileIds().size(); j++) {
@ -124,11 +113,7 @@ public class ReportGenerationService {
var isLastFile = j == reportMessage.getFileIds().size() - 1;
for (MultiFileWorkbook multiFileWorkbook : multiFileWorkbooks) {
if (excelModel == null) {
excelModel = excelTemplateReportGenerationService.calculateExcelModel(multiFileWorkbook.getReadWorkBook()
.getSheetAt(0));
}
excelTemplateReportGenerationService.generateExcelReport(reportEntries, placeholderModel, multiFileWorkbook.getTemplateName(), multiFileWorkbook.getWriteWorkbook(), dossier.getDossierName(), fileStatus, excelModel, isLastFile);
excelTemplateReportGenerationService.generateExcelReport(reportEntries, placeholderModel, multiFileWorkbook.getTemplateName(), multiFileWorkbook.getWriteWorkbook(), dossier.getDossierName(), fileStatus, multiFileWorkbook.getExcelModel(), isLastFile);
}
for (MultiFileDocument multiFileDocument : multiFileDocuments) {

135
redaction-report-service-v1/redaction-report-service-server-v1/src/test/java/com/iqser/red/service/redaction/report/v1/server/RedactionReportV2IntegrationTest.java
View File

@ -1,31 +1,5 @@
package com.iqser.red.service.redaction.report.v1.server;
import static com.iqser.red.service.redaction.report.v1.server.utils.OsUtils.getTemporaryDirectory;
import static org.mockito.Mockito.when;
import java.io.FileOutputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.commons.io.IOUtils;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Import;
import org.springframework.context.annotation.Primary;
import org.springframework.core.io.ClassPathResource;
import org.springframework.test.context.junit4.SpringRunner;
import com.amazonaws.services.s3.AmazonS3;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
@ -39,23 +13,40 @@ import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.do
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel;
import com.iqser.red.service.redaction.report.v1.api.model.ReportRequestMessage;
import com.iqser.red.service.redaction.report.v1.api.model.StoredFileInformation;
import com.iqser.red.service.redaction.report.v1.server.client.DictionaryClient;
import com.iqser.red.service.redaction.report.v1.server.client.DossierAttributesClient;
import com.iqser.red.service.redaction.report.v1.server.client.DossierAttributesConfigClient;
import com.iqser.red.service.redaction.report.v1.server.client.DossierClient;
import com.iqser.red.service.redaction.report.v1.server.client.FileAttributesConfigClient;
import com.iqser.red.service.redaction.report.v1.server.client.FileStatusClient;
import com.iqser.red.service.redaction.report.v1.server.client.RedactionLogClient;
import com.iqser.red.service.redaction.report.v1.server.client.ReportTemplateClient;
import com.iqser.red.service.redaction.report.v1.server.client.*;
import com.iqser.red.service.redaction.report.v1.server.configuration.MessagingConfiguration;
import com.iqser.red.service.redaction.report.v1.server.service.ReportGenerationService;
import com.iqser.red.service.redaction.report.v1.server.utils.FileSystemBackedStorageService;
import com.iqser.red.service.redaction.v1.model.RedactionLog;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.IOUtils;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.context.annotation.*;
import org.springframework.core.io.ClassPathResource;
import org.springframework.test.context.junit4.SpringRunner;
import org.springframework.util.Base64Utils;
import java.io.FileOutputStream;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import static com.iqser.red.service.redaction.report.v1.server.utils.OsUtils.getTemporaryDirectory;
import static org.mockito.Mockito.when;
@Slf4j
@RunWith(SpringRunner.class)
@Import(RedactionReportV2IntegrationTest.TestConfiguration.class)
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
@ -103,27 +94,15 @@ public class RedactionReportV2IntegrationTest {
@Autowired
private FileSystemBackedStorageService fileSystemBackedStorageService;
@Test
@SneakyThrows
public void testEntireReportFlow() {
@SneakyThrows
private ReportRequestMessage prepareFlow(String... templates) {
var testDossier = new Dossier();
testDossier.setDossierName("Test Dossier");
testDossier.setDossierTemplateId("dossierTemplateId");
testDossier.setId("dossierId");
when(dossierClient.getDossierById("dossierId", true, false)).thenReturn(testDossier);
ReportTemplate testTemplate = new ReportTemplate();
testTemplate.setDossierTemplateId("dossierTemplateId");
testTemplate.setTemplateId("templateId");
testTemplate.setFileName("test-template.xlsx");
testTemplate.setStorageId("templateStorageId");
testTemplate.setMultiFileReport(true);
when(reportTemplateClient.getReportTemplate("dossierTemplateId", "templateId")).thenReturn(testTemplate);
ClassPathResource templateResource = new ClassPathResource("templates/Excel Report.xlsx");
fileSystemBackedStorageService.storeObject("templateStorageId", templateResource.getInputStream());
var testDossierAttributes = new ArrayList<DossierAttribute>();
testDossierAttributes.add(DossierAttribute.builder().dossierId("dossierId").dossierAttributeConfigId("testDossierAttribute").value("Dossier Attribute Value").build());
@ -134,7 +113,6 @@ public class RedactionReportV2IntegrationTest {
.placeholder("{{dossier.attribute.TestDossierAttribute}}").dossierTemplateId("dossierTemplateId").id("testDossierAttribute").build());
when(dossierAttributesConfigClient.getDossierAttributes("dossierTemplateId")).thenReturn(dossierAttributeConfig);
var fileAttributeConfig = new ArrayList<FileAttributeConfig>();
fileAttributeConfig.add(FileAttributeConfig.builder().type(FileAttributeType.TEXT)
.placeholder("{{file.attribute.TestFileAttribute}}").dossierTemplateId("dossierTemplateId").id("testFileAttribute").build());
@ -148,29 +126,76 @@ public class RedactionReportV2IntegrationTest {
when(redactionLogClient.getRedactionLog("dossierId", "fileId", new ArrayList<>(), true, false)).thenReturn(redactionLog);
var templateIds = new HashSet<String>();
for (var template : templates) {
var templateId = Base64Utils.encodeToString(template.getBytes(StandardCharsets.UTF_8));
templateIds.add(templateId);
log.info("Assigned template Id {} to file {}", templateId, template);
ReportTemplate testTemplate = new ReportTemplate();
testTemplate.setDossierTemplateId("dossierTemplateId");
testTemplate.setTemplateId(templateId);
testTemplate.setFileName(template);
testTemplate.setStorageId(templateId + "Storage");
testTemplate.setMultiFileReport(true);
when(reportTemplateClient.getReportTemplate("dossierTemplateId", templateId)).thenReturn(testTemplate);
ClassPathResource templateResource = new ClassPathResource(template);
fileSystemBackedStorageService.storeObject(templateId + "Storage", templateResource.getInputStream());
}
var request = new ReportRequestMessage();
request.setDossierId("dossierId");
request.setFileIds(List.of("fileId"));
request.setDossierTemplateId("dossierTemplateId");
request.setDownloadId("downloadId");
request.setUserId("userId");
request.setTemplateIds(Set.of("templateId"));
var id = reportGenerationService.generateReports(request);
request.setTemplateIds(templateIds);
return request;
}
@SneakyThrows
private void processRequest(ReportRequestMessage reportRequestMessage) {
var id = reportGenerationService.generateReports(reportRequestMessage);
var object = fileSystemBackedStorageService.getObject(id);
var infoList =objectMapper.readValue(object.getInputStream(), new TypeReference<List<StoredFileInformation>>() {
var infoList = objectMapper.readValue(object.getInputStream(), new TypeReference<List<StoredFileInformation>>() {
});
for(var fileInfo: infoList) {
int i = 1;
for (var fileInfo : infoList) {
var report = fileSystemBackedStorageService.getObject(fileInfo.getStorageId());
byte[] reportInfo = IOUtils.toByteArray(report.getInputStream());
try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/output.xlsx")) {
try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/output-" + fileInfo.getTemplateId() + ".xlsx")) {
fileOutputStream.write(reportInfo);
}
System.out.println("Created temporary output file: " + getTemporaryDirectory() + "/output-" + fileInfo.getTemplateId() + ".xlsx");
i++;
}
}
@Test
@SneakyThrows
public void testBasicExcelReportFlow() {
var reportRequestMessage = prepareFlow("templates/Excel Report.xlsx");
processRequest(reportRequestMessage);
}
@Test
@SneakyThrows
public void test2FilesConcurrently() {
var reportRequestMessage = prepareFlow("templates/report.xlsx", "templates/report-advanced.xlsx");
processRequest(reportRequestMessage);
}
@Configuration
@EnableAutoConfiguration(exclude = {StorageAutoConfiguration.class, RabbitAutoConfiguration.class})
@ComponentScan("com.iqser.red.service.persistence")

BIN
redaction-report-service-v1/redaction-report-service-server-v1/src/test/resources/templates/report-advanced.xlsx Normal file
View File

Binary file not shown.

BIN
redaction-report-service-v1/redaction-report-service-server-v1/src/test/resources/templates/report.xlsx Normal file
View File

Binary file not shown.