Marmelator/redaction-report-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 129 Wiki Activity

Pull request #175: RED-4610: Split files by numberOfChars

Browse Source 
Merge in RED/redaction-report-service from RED-4610-b to master

* commit '7af205562894d16b45750e99f9da2841b5d59006':
  RED-4610: Split files by numberOfChars
This commit is contained in:
Ali Oezyetimoglu 2022-09-14 17:08:14 +02:00
commit 9b9fc94f2c
5 changed files with 141 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/model/MultiFileDocument.java
View File

@ -12,7 +12,7 @@ public class MultiFileDocument {
private XWPFDocument document;
private String templateId;
private String templateName;
private int numberOfEntries;
private int numberOfChars;
private int documentPartNr;
}

39
redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ReportGenerationService.java
View File

@ -57,8 +57,8 @@ public class ReportGenerationService {
Dossier dossier = dossierClient.getDossierById(reportMessage.getDossierId(), true, false);
List<ReportTemplate> singleFilesTemplates = new ArrayList<>();
List<MultiFileWorkbook> multiFileWorkbooks = new ArrayList<>();
List<MultiFileDocument> multiFileDocuments = new ArrayList<>();
List<MultiFileWorkbook> multiFileWorkbookReportTemplates = new ArrayList<>();
List<MultiFileDocument> multiFileDocumentReportTemplates = new ArrayList<>();
for (String templateId : reportMessage.getTemplateIds()) {
try {
ReportTemplate reportTemplate = reportTemplateClient.getReportTemplate(reportMessage.getDossierTemplateId(), templateId);
@ -73,7 +73,7 @@ public class ReportGenerationService {
}
MultiFileWorkbook multiFileWorkbook = new MultiFileWorkbook(readWorkbook, writeWorkbook, templateId, reportTemplate.getFileName(),
excelTemplateReportGenerationService.calculateExcelModel(readWorkbook.getSheetAt(0)));
multiFileWorkbooks.add(multiFileWorkbook);
multiFileWorkbookReportTemplates.add(multiFileWorkbook);
} catch (IOException e) {
throw new RuntimeException("Could not generate multifile excel report.");
}
@ -82,7 +82,7 @@ public class ReportGenerationService {
try (ByteArrayInputStream is = new ByteArrayInputStream(wordTemplate)) {
XWPFDocument doc = new XWPFDocument(is);
MultiFileDocument multiFileDocument = new MultiFileDocument(wordTemplate, doc, templateId, reportTemplate.getFileName(), 0, 0);
multiFileDocuments.add(multiFileDocument);
multiFileDocumentReportTemplates.add(multiFileDocument);
} catch (IOException e) {
throw new RuntimeException("Could not generate multifile word report.");
}
@ -112,26 +112,15 @@ public class ReportGenerationService {
var isLastFile = j == reportMessage.getFileIds().size() - 1;
for (MultiFileWorkbook multiFileWorkbook : multiFileWorkbooks) {
for (MultiFileWorkbook multiFileWorkbook : multiFileWorkbookReportTemplates) {
excelTemplateReportGenerationService.generateExcelReport(reportEntries, placeholderModel, multiFileWorkbook.getTemplateName(), multiFileWorkbook.getWriteWorkbook(), dossier.getDossierName(), fileStatus, multiFileWorkbook.getExcelModel(), isLastFile);
}
for (MultiFileDocument multiFileDocument : multiFileDocuments) {
for (MultiFileDocument multiFileDocument : multiFileDocumentReportTemplates) {
var numberOfChars = wordReportGenerationService.approxNumberOfChars(reportEntries.stream().findFirst(), fileStatus.getFilename());
if (reportEntries.size() >= reportTemplateSettings.getMultiFileChunkSize()) {
try (ByteArrayInputStream is = new ByteArrayInputStream(multiFileDocument.getTemplateAsBytes())) {
XWPFDocument doc = new XWPFDocument(is);
wordReportGenerationService.generateWordReport(reportEntries, placeholderModel, multiFileDocument.getTemplateName(), doc, fileStatus, dossier, true);
byte[] wordDoc = wordReportGenerationService.toByteArray(doc);
String storageId = reportStorageService.storeObject(reportMessage.getDownloadId(), wordDoc);
storedFileInformation.add(new StoredFileInformation(null, storageId, ReportType.WORD_TEMPLATE_MULTI_FILE, multiFileDocument.getTemplateId(), multiFileDocument.getDocumentPartNr()));
multiFileDocument.setDocumentPartNr(multiFileDocument.getDocumentPartNr() + 1);
}
continue;
}
if (multiFileDocument.getNumberOfEntries() >= reportTemplateSettings.getMultiFileChunkSize()) {
if (multiFileDocument.getNumberOfChars() >= reportTemplateSettings.getMultiFileChunkSize()) {
wordReportGenerationService.removePlaceholdersRow(wordReportGenerationService.getRedactionTable(multiFileDocument.getDocument()));
byte[] wordDoc = wordReportGenerationService.toByteArray(multiFileDocument.getDocument());
String storageId = reportStorageService.storeObject(reportMessage.getDownloadId(), wordDoc);
storedFileInformation.add(new StoredFileInformation(null, storageId, ReportType.WORD_TEMPLATE_MULTI_FILE, multiFileDocument.getTemplateId(), multiFileDocument.getDocumentPartNr()));
@ -140,11 +129,11 @@ public class ReportGenerationService {
try (ByteArrayInputStream is = new ByteArrayInputStream(multiFileDocument.getTemplateAsBytes())) {
XWPFDocument doc = new XWPFDocument(is);
multiFileDocument.setDocument(doc);
multiFileDocument.setNumberOfEntries(0);
multiFileDocument.setNumberOfChars(0);
}
}
wordReportGenerationService.generateWordReport(reportEntries, placeholderModel, multiFileDocument.getTemplateName(), multiFileDocument.getDocument(), fileStatus, dossier, isLastFile ? isLastFile : multiFileDocument.getNumberOfEntries() + reportEntries.size() >= reportTemplateSettings.getMultiFileChunkSize());
multiFileDocument.setNumberOfEntries(multiFileDocument.getNumberOfEntries() + reportEntries.size());
numberOfChars = wordReportGenerationService.generateWordReport(reportEntries, placeholderModel, multiFileDocument.getTemplateName(), multiFileDocument.getDocument(), fileStatus, dossier, isLastFile ? isLastFile : multiFileDocument.getNumberOfChars() + numberOfChars >= reportTemplateSettings.getMultiFileChunkSize());
multiFileDocument.setNumberOfChars(multiFileDocument.getNumberOfChars() + numberOfChars);
}
for (ReportTemplate reportTemplate : singleFilesTemplates) {
@ -158,13 +147,13 @@ public class ReportGenerationService {
i++;
}
for (MultiFileWorkbook multiFileWorkbook : multiFileWorkbooks) {
for (MultiFileWorkbook multiFileWorkbook : multiFileWorkbookReportTemplates) {
byte[] template = excelTemplateReportGenerationService.toByteArray(multiFileWorkbook.getWriteWorkbook());
String storageId = reportStorageService.storeObject(reportMessage.getDownloadId(), template);
storedFileInformation.add(new StoredFileInformation(null, storageId, ReportType.EXCEL_TEMPLATE_MULTI_FILE, multiFileWorkbook.getTemplateId(), 0));
}
for (MultiFileDocument multiFileDocument : multiFileDocuments) {
for (MultiFileDocument multiFileDocument : multiFileDocumentReportTemplates) {
byte[] template = wordReportGenerationService.toByteArray(multiFileDocument.getDocument());
String storageId = reportStorageService.storeObject(reportMessage.getDownloadId(), template);
storedFileInformation.add(new StoredFileInformation(null, storageId, ReportType.WORD_TEMPLATE_MULTI_FILE, multiFileDocument.getTemplateId(), multiFileDocument.getDocumentPartNr()));

100
redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/WordReportGenerationService.java
View File

@ -36,6 +36,7 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
@ -81,11 +82,13 @@ public class WordReportGenerationService {
@Timed("redactmanager_generateWordReport")
public void generateWordReport(List<ReportRedactionEntry> reportEntries, PlaceholderModel placeholderModel, String reportTemplateName, XWPFDocument doc, FileModel fileModel,
Dossier dossier, boolean isLastFile) {
public int generateWordReport(List<ReportRedactionEntry> reportEntries, PlaceholderModel placeholderModel, String reportTemplateName, XWPFDocument doc, FileModel fileModel,
Dossier dossier, boolean isLastFile) {
long start = System.currentTimeMillis();
int sumOfChars = 0;
try {
for (ImagePlaceholder imagePlaceholder : placeholderModel.getImagePlaceholders()) {
@ -96,25 +99,27 @@ public class WordReportGenerationService {
var placeholderFunctions = computePlaceholderPos(table, placeholderModel);
addRedactionEntryRows(table, reportEntries, fileModel.getFilename(), placeholderModel, placeholderFunctions);
sumOfChars += addRedactionEntryRows(table, reportEntries, fileModel.getFilename(), placeholderModel, placeholderFunctions);
replaceTextPlaceholders(doc, placeholderModel, dossier.getDossierName(), fileModel.getFilename(), table, reportEntries);
sumOfChars += replaceTextPlaceholders(doc, placeholderModel, dossier.getDossierName(), fileModel.getFilename(), table, reportEntries);
if (isLastFile) {
removePlaceholdersRow(table);
}
log.info("Report Generation took: {} for file with id {}, pageCount: {}, redactionLogEntryCount: {}, reportName: {}, className: {}", System.currentTimeMillis() - start, fileModel.getId(), fileModel.getNumberOfPages(), reportEntries.size(), reportTemplateName, getClass().getSimpleName());
log.info("Report Generation took: {} for file with id {}, pageCount: {}, redactionLogEntryCount: {}, reportName: {}, className: {}", System.currentTimeMillis() - start,
fileModel.getId(), fileModel.getNumberOfPages(), reportEntries.size(), reportTemplateName, getClass().getSimpleName());
} catch (Exception e) {
log.error(e.getMessage() + " in file: " + fileModel.getFilename());
throw new RuntimeException(e);
}
return sumOfChars;
}
private void removePlaceholdersRow(XWPFTable table) {
protected void removePlaceholdersRow(XWPFTable table) {
if (table == null) {
return;
@ -205,8 +210,10 @@ public class WordReportGenerationService {
@Timed("redactmanager_word-replaceTextPlaceholders")
public void replaceTextPlaceholders(XWPFDocument doc, PlaceholderModel placeholderModel, String dossierName, String fileName, XWPFTable tableToSkip,
List<ReportRedactionEntry> reportRedactionEntries) {
public int replaceTextPlaceholders(XWPFDocument doc, PlaceholderModel placeholderModel, String dossierName, String fileName, XWPFTable tableToSkip,
List<ReportRedactionEntry> reportRedactionEntries) {
int sumOfChars = 0;
Map<String, String> placeHolderValueMap = new HashMap<>();
for (String placeholder : placeholderModel.getPlaceholders()) {
@ -222,14 +229,18 @@ public class WordReportGenerationService {
}
for (XWPFTableRow row : tbl.getRows()) {
for (XWPFTableCell cell : row.getTableCells()) {
replacePlaceholderInParagraph(cell.getParagraphs(), placeHolderValueMap);
sumOfChars += replacePlaceholderInParagraph(cell.getParagraphs(), placeHolderValueMap);
}
}
}
return sumOfChars;
}
private void replacePlaceholderInParagraph(List<XWPFParagraph> paragraphs, Map<String, String> placeholderValueMap) {
private int replacePlaceholderInParagraph(List<XWPFParagraph> paragraphs, Map<String, String> placeholderValueMap) {
int sumOfChars = 0;
for (XWPFParagraph p : paragraphs) {
String paragraphText = p.getText();
@ -246,6 +257,7 @@ public class WordReportGenerationService {
paragraphText = paragraphText.replaceAll(escapedSearch, escapedReplace);
XWPFRun run = p.getRuns().get(0);
run.setText(paragraphText, 0);
sumOfChars += paragraphText.length();
} catch (Exception e) {
log.error("Could not replace {} with {}", escapedSearch, escapedReplace);
throw new RuntimeException(String.format("Could not replace %s with %s", escapedSearch, escapedReplace), e);
@ -264,6 +276,7 @@ public class WordReportGenerationService {
String textForLine = stringsOnNewLines[i];
ColoredText coloredText = getColor(textForLine);
newRun.setText(coloredText.getText());
sumOfChars += coloredText.getText().length();
if (coloredText.getColor() != null) {
newRun.setTextHighlightColor(coloredText.getColor());
}
@ -274,14 +287,19 @@ public class WordReportGenerationService {
}
}
}
return sumOfChars;
}
private XWPFTable getRedactionTable(XWPFDocument doc) {
protected XWPFTable getRedactionTable(XWPFDocument doc) {
for (XWPFTable tbl : doc.getTables()) {
String tblText = tbl.getText();
if (tblText.contains(PAGE_PLACEHOLDER) || tblText.contains(PARAGRAPH_PLACEHOLDER) || tblText.contains(JUSTIFICATION_PLACEHOLDER) || tblText.contains(EXCERPT_PLACEHOLDER) || tblText.contains(JUSTIFICATION_PARAGRAPH_PLACEHOLDER) || tblText.contains(JUSTIFICATION_REASON_PLACEHOLDER) || tblText.contains(JUSTIFICATION_LEGAL_BASIS_PLACEHOLDER) || tblText.contains(JUSTIFICATION_TEXT_PLACEHOLDER) || tblText.contains(SEEDS_FUNCTION_REDACTION_GROUPED_BY_JUSTIFICATION_PAGES_PLACEHOLDER) || tblText.contains(SEEDS_FUNCTION_JUSTIFICATION_PLACEHOLDER)) {
if (tblText.contains(PAGE_PLACEHOLDER) || tblText.contains(PARAGRAPH_PLACEHOLDER) || tblText.contains(JUSTIFICATION_PLACEHOLDER) || tblText.contains(
EXCERPT_PLACEHOLDER) || tblText.contains(JUSTIFICATION_PARAGRAPH_PLACEHOLDER) || tblText.contains(JUSTIFICATION_REASON_PLACEHOLDER) || tblText.contains(
JUSTIFICATION_LEGAL_BASIS_PLACEHOLDER) || tblText.contains(JUSTIFICATION_TEXT_PLACEHOLDER) || tblText.contains(
SEEDS_FUNCTION_REDACTION_GROUPED_BY_JUSTIFICATION_PAGES_PLACEHOLDER) || tblText.contains(SEEDS_FUNCTION_JUSTIFICATION_PLACEHOLDER)) {
return tbl;
}
}
@ -291,21 +309,12 @@ public class WordReportGenerationService {
private boolean containsRedactionPlaceholder(String text) {
return text.startsWith(DOSSIER_ATTRIBUTE_PLACEHOLDER_BASE) ||
text.startsWith(FILE_ATTRIBUTE_PLACEHOLDER_BASE) ||
text.contains(FILE_NAME_PLACEHOLDER) ||
text.contains(PAGE_PLACEHOLDER) ||
text.contains(PARAGRAPH_PLACEHOLDER) ||
text.contains(JUSTIFICATION_PLACEHOLDER) ||
text.contains(EXCERPT_PLACEHOLDER) ||
text.contains(JUSTIFICATION_PARAGRAPH_PLACEHOLDER) ||
text.contains(JUSTIFICATION_REASON_PLACEHOLDER) ||
text.contains(REDACTION_VALUE_PLACEHOLDER) ||
text.contains(JUSTIFICATION_LEGAL_BASIS_PLACEHOLDER) ||
text.contains(JUSTIFICATION_TEXT_PLACEHOLDER) ||
text.contains(SEEDS_FUNCTION_REDACTION_GROUPED_BY_JUSTIFICATION_PAGES_PLACEHOLDER) ||
text.contains(SEEDS_FUNCTION_JUSTIFICATION_PLACEHOLDER) ||
text.contains(REDACTION_ENTITY_DISPLAY_NAME_PLACEHOLDER);
return text.startsWith(DOSSIER_ATTRIBUTE_PLACEHOLDER_BASE) || text.startsWith(FILE_ATTRIBUTE_PLACEHOLDER_BASE) || text.contains(FILE_NAME_PLACEHOLDER) || text.contains(
PAGE_PLACEHOLDER) || text.contains(PARAGRAPH_PLACEHOLDER) || text.contains(JUSTIFICATION_PLACEHOLDER) || text.contains(EXCERPT_PLACEHOLDER) || text.contains(
JUSTIFICATION_PARAGRAPH_PLACEHOLDER) || text.contains(JUSTIFICATION_REASON_PLACEHOLDER) || text.contains(REDACTION_VALUE_PLACEHOLDER) || text.contains(
JUSTIFICATION_LEGAL_BASIS_PLACEHOLDER) || text.contains(JUSTIFICATION_TEXT_PLACEHOLDER) || text.contains(
SEEDS_FUNCTION_REDACTION_GROUPED_BY_JUSTIFICATION_PAGES_PLACEHOLDER) || text.contains(SEEDS_FUNCTION_JUSTIFICATION_PLACEHOLDER) || text.contains(
REDACTION_ENTITY_DISPLAY_NAME_PLACEHOLDER);
}
@ -332,11 +341,13 @@ public class WordReportGenerationService {
@Timed("redactmanager_word-addRedactionEntryRows")
private void addRedactionEntryRows(XWPFTable table, List<ReportRedactionEntry> reportEntries, String filename, PlaceholderModel placeholderModel,
PlaceHolderFunctions placeHolderFunctions) {
private int addRedactionEntryRows(XWPFTable table, List<ReportRedactionEntry> reportEntries, String filename, PlaceholderModel placeholderModel,
PlaceHolderFunctions placeHolderFunctions) {
int sumOfChars = 0;
if (table == null) {
return;
return sumOfChars;
}
if (placeHolderFunctions.getFoundPlaceHolder().contains(SEEDS_FUNCTION_REDACTION_GROUPED_BY_JUSTIFICATION_PAGES_PLACEHOLDER)) {
@ -346,17 +357,19 @@ public class WordReportGenerationService {
for (Map.Entry<String, List<ReportRedactionEntry>> entry : redactionsPerJustification.entrySet()) {
XWPFTableRow row = table.createRow();
for (Map.Entry<Integer, Function<PlaceholderInput, String>> entry1 : placeHolderFunctions.getFunctionPerPlaceHolder().entrySet()) {
setText(row.getCell(entry1.getKey()), entry1.getValue().apply(new PlaceholderInput(filename, null, placeholderModel, entry)));
sumOfChars += setText(row.getCell(entry1.getKey()), entry1.getValue().apply(new PlaceholderInput(filename, null, placeholderModel, entry)));
}
}
} else {
reportEntries.forEach(entry -> {
for (var entry : reportEntries) {
XWPFTableRow row = table.createRow();
for (Map.Entry<Integer, Function<PlaceholderInput, String>> entry1 : placeHolderFunctions.getFunctionPerPlaceHolder().entrySet()) {
setText(row.getCell(entry1.getKey()), entry1.getValue().apply(new PlaceholderInput(filename, entry, placeholderModel, null)));
sumOfChars += setText(row.getCell(entry1.getKey()), entry1.getValue().apply(new PlaceholderInput(filename, entry, placeholderModel, null)));
}
});
}
}
return sumOfChars;
}
@ -485,13 +498,14 @@ public class WordReportGenerationService {
}
private void setText(XWPFTableCell cell, String value) {
private int setText(XWPFTableCell cell, String value) {
cell.setVerticalAlignment(XWPFTableCell.XWPFVertAlign.CENTER);
XWPFParagraph addParagraph = cell.addParagraph();
XWPFRun run = addParagraph.createRun();
run.setFontSize(10);
run.setText(value);
return value.length();
}
@ -512,6 +526,22 @@ public class WordReportGenerationService {
}
protected int approxNumberOfChars(Optional<ReportRedactionEntry> firstReportEntry, String filename) {
if(firstReportEntry.isPresent()) {
int sumOfChars = 0;
sumOfChars += filename.length();
sumOfChars += String.valueOf(firstReportEntry.get().getPage()).length();
sumOfChars += firstReportEntry.get().getJustificationParagraph().length();
sumOfChars += firstReportEntry.get().getJustification().length();
return sumOfChars;
}
return 0;
}
@SneakyThrows
public byte[] toByteArray(XWPFDocument doc) {

2
redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/settings/ReportTemplateSettings.java
View File

@ -8,6 +8,6 @@ import lombok.Data;
@ConfigurationProperties("redaction-report-service")
public class ReportTemplateSettings {
private int multiFileChunkSize = 2000;
private int multiFileChunkSize = 1000000;
}

82
redaction-report-service-v1/redaction-report-service-server-v1/src/test/java/com/iqser/red/service/redaction/report/v1/server/RedactionReportV2IntegrationTest.java
View File

@ -20,8 +20,10 @@ import com.iqser.red.service.redaction.report.v1.server.utils.FileSystemBackedSt
import com.iqser.red.service.redaction.v1.model.RedactionLog;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.IOUtils;
import org.junit.Test;
import org.junit.runner.RunWith;
@ -96,7 +98,8 @@ public class RedactionReportV2IntegrationTest {
@SneakyThrows
private ReportRequestMessage prepareFlow(String... templates) {
private ReportRequestMessage prepareFlow(int numOfFiles, String... templates) {
var testDossier = new Dossier();
testDossier.setDossierName("Test Dossier");
testDossier.setDossierTemplateId("dossierTemplateId");
@ -109,23 +112,29 @@ public class RedactionReportV2IntegrationTest {
when(dossierAttributesClient.getDossierAttributes("dossierId")).thenReturn(testDossierAttributes);
var dossierAttributeConfig = new ArrayList<DossierAttributeConfig>();
dossierAttributeConfig.add(DossierAttributeConfig.builder().type(DossierAttributeType.TEXT)
.placeholder("{{dossier.attribute.TestDossierAttribute}}").dossierTemplateId("dossierTemplateId").id("testDossierAttribute").build());
dossierAttributeConfig.add(DossierAttributeConfig.builder()
.type(DossierAttributeType.TEXT)
.placeholder("{{dossier.attribute.TestDossierAttribute}}")
.dossierTemplateId("dossierTemplateId")
.id("testDossierAttribute")
.build());
when(dossierAttributesConfigClient.getDossierAttributes("dossierTemplateId")).thenReturn(dossierAttributeConfig);
var fileAttributeConfig = new ArrayList<FileAttributeConfig>();
fileAttributeConfig.add(FileAttributeConfig.builder().type(FileAttributeType.TEXT)
.placeholder("{{file.attribute.TestFileAttribute}}").dossierTemplateId("dossierTemplateId").id("testFileAttribute").build());
fileAttributeConfig.add(FileAttributeConfig.builder()
.type(FileAttributeType.TEXT)
.placeholder("{{file.attribute.TestFileAttribute}}")
.dossierTemplateId("dossierTemplateId")
.id("testFileAttribute")
.build());
when(fileAttributesConfigClient.getFileAttributeConfigs("dossierTemplateId")).thenReturn(fileAttributeConfig);
var fileModel = FileModel.builder().filename("filename").build();
fileModel.setFileAttributes(Map.of("testFileAttribute", "File Attribute Value"));
when(fileStatusClient.getFileStatus("dossierId", "fileId")).thenReturn(fileModel);
var redactionLog = objectMapper.readValue(new ClassPathResource("files/redactionLog.json").getInputStream(), RedactionLog.class);
when(redactionLogClient.getRedactionLog("dossierId", "fileId", new ArrayList<>(), true, false)).thenReturn(redactionLog);
var fileModels = createFileModels(numOfFiles);
for (int i = 1; i <= numOfFiles; i++) {
when(fileStatusClient.getFileStatus("dossierId", "fileId" + i)).thenReturn(fileModels.get(i - 1));
when(redactionLogClient.getRedactionLog("dossierId", "fileId" + i, new ArrayList<>(), true, false)).thenReturn(redactionLog);
}
var templateIds = new HashSet<String>();
for (var template : templates) {
@ -143,13 +152,14 @@ public class RedactionReportV2IntegrationTest {
ClassPathResource templateResource = new ClassPathResource(template);
fileSystemBackedStorageService.storeObject(templateId + "Storage", templateResource.getInputStream());
}
var fileIds = new ArrayList<String>();
fileModels.forEach(fileModel -> fileIds.add(fileModel.getId()));
var request = new ReportRequestMessage();
request.setDossierId("dossierId");
request.setFileIds(List.of("fileId"));
request.setFileIds(fileIds);
request.setDossierTemplateId("dossierTemplateId");
request.setDownloadId("downloadId");
request.setUserId("userId");
@ -158,44 +168,58 @@ public class RedactionReportV2IntegrationTest {
return request;
}
@SneakyThrows
private void processRequest(ReportRequestMessage reportRequestMessage) {
private void processRequest(ReportRequestMessage reportRequestMessage, String fileEnding) {
var id = reportGenerationService.generateReports(reportRequestMessage);
var object = fileSystemBackedStorageService.getObject(id);
var infoList = objectMapper.readValue(object.getInputStream(), new TypeReference<List<StoredFileInformation>>() {
});
int i = 1;
for (var fileInfo : infoList) {
var report = fileSystemBackedStorageService.getObject(fileInfo.getStorageId());
byte[] reportInfo = IOUtils.toByteArray(report.getInputStream());
try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/output-" + fileInfo.getTemplateId() + ".xlsx")) {
var outputPath = getTemporaryDirectory() + "/output-" + fileInfo.getTemplateId() + "_" + i + fileEnding;
try (FileOutputStream fileOutputStream = new FileOutputStream(outputPath)) {
fileOutputStream.write(reportInfo);
}
System.out.println("Created temporary output file: " + getTemporaryDirectory() + "/output-" + fileInfo.getTemplateId() + ".xlsx");
System.out.println("Created temporary output file: " + outputPath);
i++;
}
}
@Test
@SneakyThrows
public void testBasicExcelReportFlow() {
var reportRequestMessage = prepareFlow("templates/Excel Report.xlsx");
processRequest(reportRequestMessage);
var reportRequestMessage = prepareFlow(1,"templates/Excel Report.xlsx");
processRequest(reportRequestMessage, ".xlsx");
}
@Test
@SneakyThrows
public void testMultifileWordReportFlow() {
var reportRequestMessage = prepareFlow(2,"templates/Justification Appendix A1.docx");
processRequest(reportRequestMessage, ".docx");
}
@Test
@SneakyThrows
public void test2FilesConcurrently() {
var reportRequestMessage = prepareFlow("templates/report.xlsx", "templates/report-advanced.xlsx");
processRequest(reportRequestMessage);
var reportRequestMessage = prepareFlow(1,"templates/report.xlsx", "templates/report-advanced.xlsx");
processRequest(reportRequestMessage, ".xlsx");
}
@Configuration
@EnableAutoConfiguration(exclude = {StorageAutoConfiguration.class, RabbitAutoConfiguration.class})
@ComponentScan("com.iqser.red.service.persistence")
@ -204,10 +228,24 @@ public class RedactionReportV2IntegrationTest {
@Bean
@Primary
public StorageService inmemoryStorage() {
return new FileSystemBackedStorageService();
}
}
private List<FileModel> createFileModels(int numOfFileModelsToCreate) {
if (numOfFileModelsToCreate <= 0) {
numOfFileModelsToCreate = 1;
}
var fileModels = new ArrayList<FileModel>();
for (int i = 1; i <= numOfFileModelsToCreate; i++) {
fileModels.add(FileModel.builder().id("fileId" + i).filename("filename" + i).fileAttributes(Map.of("testFileAttribute" + i, "File Attribute Value" + i)).build());
}
return fileModels;
}
}