From 7af205562894d16b45750e99f9da2841b5d59006 Mon Sep 17 00:00:00 2001 From: Ali Oezyetimoglu Date: Tue, 13 Sep 2022 09:12:53 +0200 Subject: [PATCH] RED-4610: Split files by numberOfChars --- .../v1/server/model/MultiFileDocument.java | 2 +- .../service/ReportGenerationService.java | 39 +++---- .../service/WordReportGenerationService.java | 100 ++++++++++++------ .../settings/ReportTemplateSettings.java | 2 +- .../RedactionReportV2IntegrationTest.java | 82 ++++++++++---- 5 files changed, 141 insertions(+), 84 deletions(-) diff --git a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/model/MultiFileDocument.java b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/model/MultiFileDocument.java index 8a632f4..2281923 100644 --- a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/model/MultiFileDocument.java +++ b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/model/MultiFileDocument.java @@ -12,7 +12,7 @@ public class MultiFileDocument { private XWPFDocument document; private String templateId; private String templateName; - private int numberOfEntries; + private int numberOfChars; private int documentPartNr; } diff --git a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ReportGenerationService.java b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ReportGenerationService.java index 629f5e2..dba01b1 100644 --- a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ReportGenerationService.java +++ b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ReportGenerationService.java @@ -57,8 +57,8 @@ public class ReportGenerationService { Dossier dossier = dossierClient.getDossierById(reportMessage.getDossierId(), true, false); List singleFilesTemplates = new ArrayList<>(); - List multiFileWorkbooks = new ArrayList<>(); - List multiFileDocuments = new ArrayList<>(); + List multiFileWorkbookReportTemplates = new ArrayList<>(); + List multiFileDocumentReportTemplates = new ArrayList<>(); for (String templateId : reportMessage.getTemplateIds()) { try { ReportTemplate reportTemplate = reportTemplateClient.getReportTemplate(reportMessage.getDossierTemplateId(), templateId); @@ -73,7 +73,7 @@ public class ReportGenerationService { } MultiFileWorkbook multiFileWorkbook = new MultiFileWorkbook(readWorkbook, writeWorkbook, templateId, reportTemplate.getFileName(), excelTemplateReportGenerationService.calculateExcelModel(readWorkbook.getSheetAt(0))); - multiFileWorkbooks.add(multiFileWorkbook); + multiFileWorkbookReportTemplates.add(multiFileWorkbook); } catch (IOException e) { throw new RuntimeException("Could not generate multifile excel report."); } @@ -82,7 +82,7 @@ public class ReportGenerationService { try (ByteArrayInputStream is = new ByteArrayInputStream(wordTemplate)) { XWPFDocument doc = new XWPFDocument(is); MultiFileDocument multiFileDocument = new MultiFileDocument(wordTemplate, doc, templateId, reportTemplate.getFileName(), 0, 0); - multiFileDocuments.add(multiFileDocument); + multiFileDocumentReportTemplates.add(multiFileDocument); } catch (IOException e) { throw new RuntimeException("Could not generate multifile word report."); } @@ -112,26 +112,15 @@ public class ReportGenerationService { var isLastFile = j == reportMessage.getFileIds().size() - 1; - for (MultiFileWorkbook multiFileWorkbook : multiFileWorkbooks) { + for (MultiFileWorkbook multiFileWorkbook : multiFileWorkbookReportTemplates) { excelTemplateReportGenerationService.generateExcelReport(reportEntries, placeholderModel, multiFileWorkbook.getTemplateName(), multiFileWorkbook.getWriteWorkbook(), dossier.getDossierName(), fileStatus, multiFileWorkbook.getExcelModel(), isLastFile); } - for (MultiFileDocument multiFileDocument : multiFileDocuments) { + for (MultiFileDocument multiFileDocument : multiFileDocumentReportTemplates) { + var numberOfChars = wordReportGenerationService.approxNumberOfChars(reportEntries.stream().findFirst(), fileStatus.getFilename()); - if (reportEntries.size() >= reportTemplateSettings.getMultiFileChunkSize()) { - - try (ByteArrayInputStream is = new ByteArrayInputStream(multiFileDocument.getTemplateAsBytes())) { - XWPFDocument doc = new XWPFDocument(is); - wordReportGenerationService.generateWordReport(reportEntries, placeholderModel, multiFileDocument.getTemplateName(), doc, fileStatus, dossier, true); - byte[] wordDoc = wordReportGenerationService.toByteArray(doc); - String storageId = reportStorageService.storeObject(reportMessage.getDownloadId(), wordDoc); - storedFileInformation.add(new StoredFileInformation(null, storageId, ReportType.WORD_TEMPLATE_MULTI_FILE, multiFileDocument.getTemplateId(), multiFileDocument.getDocumentPartNr())); - multiFileDocument.setDocumentPartNr(multiFileDocument.getDocumentPartNr() + 1); - } - continue; - } - - if (multiFileDocument.getNumberOfEntries() >= reportTemplateSettings.getMultiFileChunkSize()) { + if (multiFileDocument.getNumberOfChars() >= reportTemplateSettings.getMultiFileChunkSize()) { + wordReportGenerationService.removePlaceholdersRow(wordReportGenerationService.getRedactionTable(multiFileDocument.getDocument())); byte[] wordDoc = wordReportGenerationService.toByteArray(multiFileDocument.getDocument()); String storageId = reportStorageService.storeObject(reportMessage.getDownloadId(), wordDoc); storedFileInformation.add(new StoredFileInformation(null, storageId, ReportType.WORD_TEMPLATE_MULTI_FILE, multiFileDocument.getTemplateId(), multiFileDocument.getDocumentPartNr())); @@ -140,11 +129,11 @@ public class ReportGenerationService { try (ByteArrayInputStream is = new ByteArrayInputStream(multiFileDocument.getTemplateAsBytes())) { XWPFDocument doc = new XWPFDocument(is); multiFileDocument.setDocument(doc); - multiFileDocument.setNumberOfEntries(0); + multiFileDocument.setNumberOfChars(0); } } - wordReportGenerationService.generateWordReport(reportEntries, placeholderModel, multiFileDocument.getTemplateName(), multiFileDocument.getDocument(), fileStatus, dossier, isLastFile ? isLastFile : multiFileDocument.getNumberOfEntries() + reportEntries.size() >= reportTemplateSettings.getMultiFileChunkSize()); - multiFileDocument.setNumberOfEntries(multiFileDocument.getNumberOfEntries() + reportEntries.size()); + numberOfChars = wordReportGenerationService.generateWordReport(reportEntries, placeholderModel, multiFileDocument.getTemplateName(), multiFileDocument.getDocument(), fileStatus, dossier, isLastFile ? isLastFile : multiFileDocument.getNumberOfChars() + numberOfChars >= reportTemplateSettings.getMultiFileChunkSize()); + multiFileDocument.setNumberOfChars(multiFileDocument.getNumberOfChars() + numberOfChars); } for (ReportTemplate reportTemplate : singleFilesTemplates) { @@ -158,13 +147,13 @@ public class ReportGenerationService { i++; } - for (MultiFileWorkbook multiFileWorkbook : multiFileWorkbooks) { + for (MultiFileWorkbook multiFileWorkbook : multiFileWorkbookReportTemplates) { byte[] template = excelTemplateReportGenerationService.toByteArray(multiFileWorkbook.getWriteWorkbook()); String storageId = reportStorageService.storeObject(reportMessage.getDownloadId(), template); storedFileInformation.add(new StoredFileInformation(null, storageId, ReportType.EXCEL_TEMPLATE_MULTI_FILE, multiFileWorkbook.getTemplateId(), 0)); } - for (MultiFileDocument multiFileDocument : multiFileDocuments) { + for (MultiFileDocument multiFileDocument : multiFileDocumentReportTemplates) { byte[] template = wordReportGenerationService.toByteArray(multiFileDocument.getDocument()); String storageId = reportStorageService.storeObject(reportMessage.getDownloadId(), template); storedFileInformation.add(new StoredFileInformation(null, storageId, ReportType.WORD_TEMPLATE_MULTI_FILE, multiFileDocument.getTemplateId(), multiFileDocument.getDocumentPartNr())); diff --git a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/WordReportGenerationService.java b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/WordReportGenerationService.java index bb59f70..ca9e898 100644 --- a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/WordReportGenerationService.java +++ b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/WordReportGenerationService.java @@ -36,6 +36,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.SortedSet; import java.util.TreeSet; @@ -81,11 +82,13 @@ public class WordReportGenerationService { @Timed("redactmanager_generateWordReport") - public void generateWordReport(List reportEntries, PlaceholderModel placeholderModel, String reportTemplateName, XWPFDocument doc, FileModel fileModel, - Dossier dossier, boolean isLastFile) { + public int generateWordReport(List reportEntries, PlaceholderModel placeholderModel, String reportTemplateName, XWPFDocument doc, FileModel fileModel, + Dossier dossier, boolean isLastFile) { long start = System.currentTimeMillis(); + int sumOfChars = 0; + try { for (ImagePlaceholder imagePlaceholder : placeholderModel.getImagePlaceholders()) { @@ -96,25 +99,27 @@ public class WordReportGenerationService { var placeholderFunctions = computePlaceholderPos(table, placeholderModel); - addRedactionEntryRows(table, reportEntries, fileModel.getFilename(), placeholderModel, placeholderFunctions); + sumOfChars += addRedactionEntryRows(table, reportEntries, fileModel.getFilename(), placeholderModel, placeholderFunctions); - replaceTextPlaceholders(doc, placeholderModel, dossier.getDossierName(), fileModel.getFilename(), table, reportEntries); + sumOfChars += replaceTextPlaceholders(doc, placeholderModel, dossier.getDossierName(), fileModel.getFilename(), table, reportEntries); if (isLastFile) { removePlaceholdersRow(table); } - log.info("Report Generation took: {} for file with id {}, pageCount: {}, redactionLogEntryCount: {}, reportName: {}, className: {}", System.currentTimeMillis() - start, fileModel.getId(), fileModel.getNumberOfPages(), reportEntries.size(), reportTemplateName, getClass().getSimpleName()); + log.info("Report Generation took: {} for file with id {}, pageCount: {}, redactionLogEntryCount: {}, reportName: {}, className: {}", System.currentTimeMillis() - start, + fileModel.getId(), fileModel.getNumberOfPages(), reportEntries.size(), reportTemplateName, getClass().getSimpleName()); } catch (Exception e) { log.error(e.getMessage() + " in file: " + fileModel.getFilename()); throw new RuntimeException(e); } + return sumOfChars; } - private void removePlaceholdersRow(XWPFTable table) { + protected void removePlaceholdersRow(XWPFTable table) { if (table == null) { return; @@ -205,8 +210,10 @@ public class WordReportGenerationService { @Timed("redactmanager_word-replaceTextPlaceholders") - public void replaceTextPlaceholders(XWPFDocument doc, PlaceholderModel placeholderModel, String dossierName, String fileName, XWPFTable tableToSkip, - List reportRedactionEntries) { + public int replaceTextPlaceholders(XWPFDocument doc, PlaceholderModel placeholderModel, String dossierName, String fileName, XWPFTable tableToSkip, + List reportRedactionEntries) { + + int sumOfChars = 0; Map placeHolderValueMap = new HashMap<>(); for (String placeholder : placeholderModel.getPlaceholders()) { @@ -222,14 +229,18 @@ public class WordReportGenerationService { } for (XWPFTableRow row : tbl.getRows()) { for (XWPFTableCell cell : row.getTableCells()) { - replacePlaceholderInParagraph(cell.getParagraphs(), placeHolderValueMap); + sumOfChars += replacePlaceholderInParagraph(cell.getParagraphs(), placeHolderValueMap); } } } + + return sumOfChars; } - private void replacePlaceholderInParagraph(List paragraphs, Map placeholderValueMap) { + private int replacePlaceholderInParagraph(List paragraphs, Map placeholderValueMap) { + + int sumOfChars = 0; for (XWPFParagraph p : paragraphs) { String paragraphText = p.getText(); @@ -246,6 +257,7 @@ public class WordReportGenerationService { paragraphText = paragraphText.replaceAll(escapedSearch, escapedReplace); XWPFRun run = p.getRuns().get(0); run.setText(paragraphText, 0); + sumOfChars += paragraphText.length(); } catch (Exception e) { log.error("Could not replace {} with {}", escapedSearch, escapedReplace); throw new RuntimeException(String.format("Could not replace %s with %s", escapedSearch, escapedReplace), e); @@ -264,6 +276,7 @@ public class WordReportGenerationService { String textForLine = stringsOnNewLines[i]; ColoredText coloredText = getColor(textForLine); newRun.setText(coloredText.getText()); + sumOfChars += coloredText.getText().length(); if (coloredText.getColor() != null) { newRun.setTextHighlightColor(coloredText.getColor()); } @@ -274,14 +287,19 @@ public class WordReportGenerationService { } } } + + return sumOfChars; } - private XWPFTable getRedactionTable(XWPFDocument doc) { + protected XWPFTable getRedactionTable(XWPFDocument doc) { for (XWPFTable tbl : doc.getTables()) { String tblText = tbl.getText(); - if (tblText.contains(PAGE_PLACEHOLDER) || tblText.contains(PARAGRAPH_PLACEHOLDER) || tblText.contains(JUSTIFICATION_PLACEHOLDER) || tblText.contains(EXCERPT_PLACEHOLDER) || tblText.contains(JUSTIFICATION_PARAGRAPH_PLACEHOLDER) || tblText.contains(JUSTIFICATION_REASON_PLACEHOLDER) || tblText.contains(JUSTIFICATION_LEGAL_BASIS_PLACEHOLDER) || tblText.contains(JUSTIFICATION_TEXT_PLACEHOLDER) || tblText.contains(SEEDS_FUNCTION_REDACTION_GROUPED_BY_JUSTIFICATION_PAGES_PLACEHOLDER) || tblText.contains(SEEDS_FUNCTION_JUSTIFICATION_PLACEHOLDER)) { + if (tblText.contains(PAGE_PLACEHOLDER) || tblText.contains(PARAGRAPH_PLACEHOLDER) || tblText.contains(JUSTIFICATION_PLACEHOLDER) || tblText.contains( + EXCERPT_PLACEHOLDER) || tblText.contains(JUSTIFICATION_PARAGRAPH_PLACEHOLDER) || tblText.contains(JUSTIFICATION_REASON_PLACEHOLDER) || tblText.contains( + JUSTIFICATION_LEGAL_BASIS_PLACEHOLDER) || tblText.contains(JUSTIFICATION_TEXT_PLACEHOLDER) || tblText.contains( + SEEDS_FUNCTION_REDACTION_GROUPED_BY_JUSTIFICATION_PAGES_PLACEHOLDER) || tblText.contains(SEEDS_FUNCTION_JUSTIFICATION_PLACEHOLDER)) { return tbl; } } @@ -291,21 +309,12 @@ public class WordReportGenerationService { private boolean containsRedactionPlaceholder(String text) { - return text.startsWith(DOSSIER_ATTRIBUTE_PLACEHOLDER_BASE) || - text.startsWith(FILE_ATTRIBUTE_PLACEHOLDER_BASE) || - text.contains(FILE_NAME_PLACEHOLDER) || - text.contains(PAGE_PLACEHOLDER) || - text.contains(PARAGRAPH_PLACEHOLDER) || - text.contains(JUSTIFICATION_PLACEHOLDER) || - text.contains(EXCERPT_PLACEHOLDER) || - text.contains(JUSTIFICATION_PARAGRAPH_PLACEHOLDER) || - text.contains(JUSTIFICATION_REASON_PLACEHOLDER) || - text.contains(REDACTION_VALUE_PLACEHOLDER) || - text.contains(JUSTIFICATION_LEGAL_BASIS_PLACEHOLDER) || - text.contains(JUSTIFICATION_TEXT_PLACEHOLDER) || - text.contains(SEEDS_FUNCTION_REDACTION_GROUPED_BY_JUSTIFICATION_PAGES_PLACEHOLDER) || - text.contains(SEEDS_FUNCTION_JUSTIFICATION_PLACEHOLDER) || - text.contains(REDACTION_ENTITY_DISPLAY_NAME_PLACEHOLDER); + return text.startsWith(DOSSIER_ATTRIBUTE_PLACEHOLDER_BASE) || text.startsWith(FILE_ATTRIBUTE_PLACEHOLDER_BASE) || text.contains(FILE_NAME_PLACEHOLDER) || text.contains( + PAGE_PLACEHOLDER) || text.contains(PARAGRAPH_PLACEHOLDER) || text.contains(JUSTIFICATION_PLACEHOLDER) || text.contains(EXCERPT_PLACEHOLDER) || text.contains( + JUSTIFICATION_PARAGRAPH_PLACEHOLDER) || text.contains(JUSTIFICATION_REASON_PLACEHOLDER) || text.contains(REDACTION_VALUE_PLACEHOLDER) || text.contains( + JUSTIFICATION_LEGAL_BASIS_PLACEHOLDER) || text.contains(JUSTIFICATION_TEXT_PLACEHOLDER) || text.contains( + SEEDS_FUNCTION_REDACTION_GROUPED_BY_JUSTIFICATION_PAGES_PLACEHOLDER) || text.contains(SEEDS_FUNCTION_JUSTIFICATION_PLACEHOLDER) || text.contains( + REDACTION_ENTITY_DISPLAY_NAME_PLACEHOLDER); } @@ -332,11 +341,13 @@ public class WordReportGenerationService { @Timed("redactmanager_word-addRedactionEntryRows") - private void addRedactionEntryRows(XWPFTable table, List reportEntries, String filename, PlaceholderModel placeholderModel, - PlaceHolderFunctions placeHolderFunctions) { + private int addRedactionEntryRows(XWPFTable table, List reportEntries, String filename, PlaceholderModel placeholderModel, + PlaceHolderFunctions placeHolderFunctions) { + + int sumOfChars = 0; if (table == null) { - return; + return sumOfChars; } if (placeHolderFunctions.getFoundPlaceHolder().contains(SEEDS_FUNCTION_REDACTION_GROUPED_BY_JUSTIFICATION_PAGES_PLACEHOLDER)) { @@ -346,17 +357,19 @@ public class WordReportGenerationService { for (Map.Entry> entry : redactionsPerJustification.entrySet()) { XWPFTableRow row = table.createRow(); for (Map.Entry> entry1 : placeHolderFunctions.getFunctionPerPlaceHolder().entrySet()) { - setText(row.getCell(entry1.getKey()), entry1.getValue().apply(new PlaceholderInput(filename, null, placeholderModel, entry))); + sumOfChars += setText(row.getCell(entry1.getKey()), entry1.getValue().apply(new PlaceholderInput(filename, null, placeholderModel, entry))); } } } else { - reportEntries.forEach(entry -> { + for (var entry : reportEntries) { XWPFTableRow row = table.createRow(); for (Map.Entry> entry1 : placeHolderFunctions.getFunctionPerPlaceHolder().entrySet()) { - setText(row.getCell(entry1.getKey()), entry1.getValue().apply(new PlaceholderInput(filename, entry, placeholderModel, null))); + sumOfChars += setText(row.getCell(entry1.getKey()), entry1.getValue().apply(new PlaceholderInput(filename, entry, placeholderModel, null))); } - }); + } } + + return sumOfChars; } @@ -485,13 +498,14 @@ public class WordReportGenerationService { } - private void setText(XWPFTableCell cell, String value) { + private int setText(XWPFTableCell cell, String value) { cell.setVerticalAlignment(XWPFTableCell.XWPFVertAlign.CENTER); XWPFParagraph addParagraph = cell.addParagraph(); XWPFRun run = addParagraph.createRun(); run.setFontSize(10); run.setText(value); + return value.length(); } @@ -512,6 +526,22 @@ public class WordReportGenerationService { } + protected int approxNumberOfChars(Optional firstReportEntry, String filename) { + if(firstReportEntry.isPresent()) { + int sumOfChars = 0; + + sumOfChars += filename.length(); + sumOfChars += String.valueOf(firstReportEntry.get().getPage()).length(); + sumOfChars += firstReportEntry.get().getJustificationParagraph().length(); + sumOfChars += firstReportEntry.get().getJustification().length(); + + return sumOfChars; + } + + return 0; + } + + @SneakyThrows public byte[] toByteArray(XWPFDocument doc) { diff --git a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/settings/ReportTemplateSettings.java b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/settings/ReportTemplateSettings.java index 6b6d8e2..712e49c 100644 --- a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/settings/ReportTemplateSettings.java +++ b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/settings/ReportTemplateSettings.java @@ -8,6 +8,6 @@ import lombok.Data; @ConfigurationProperties("redaction-report-service") public class ReportTemplateSettings { - private int multiFileChunkSize = 2000; + private int multiFileChunkSize = 1000000; } \ No newline at end of file diff --git a/redaction-report-service-v1/redaction-report-service-server-v1/src/test/java/com/iqser/red/service/redaction/report/v1/server/RedactionReportV2IntegrationTest.java b/redaction-report-service-v1/redaction-report-service-server-v1/src/test/java/com/iqser/red/service/redaction/report/v1/server/RedactionReportV2IntegrationTest.java index 13e5a66..5753faf 100644 --- a/redaction-report-service-v1/redaction-report-service-server-v1/src/test/java/com/iqser/red/service/redaction/report/v1/server/RedactionReportV2IntegrationTest.java +++ b/redaction-report-service-v1/redaction-report-service-server-v1/src/test/java/com/iqser/red/service/redaction/report/v1/server/RedactionReportV2IntegrationTest.java @@ -20,8 +20,10 @@ import com.iqser.red.service.redaction.report.v1.server.utils.FileSystemBackedSt import com.iqser.red.service.redaction.v1.model.RedactionLog; import com.iqser.red.storage.commons.StorageAutoConfiguration; import com.iqser.red.storage.commons.service.StorageService; + import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; + import org.apache.commons.io.IOUtils; import org.junit.Test; import org.junit.runner.RunWith; @@ -96,7 +98,8 @@ public class RedactionReportV2IntegrationTest { @SneakyThrows - private ReportRequestMessage prepareFlow(String... templates) { + private ReportRequestMessage prepareFlow(int numOfFiles, String... templates) { + var testDossier = new Dossier(); testDossier.setDossierName("Test Dossier"); testDossier.setDossierTemplateId("dossierTemplateId"); @@ -109,23 +112,29 @@ public class RedactionReportV2IntegrationTest { when(dossierAttributesClient.getDossierAttributes("dossierId")).thenReturn(testDossierAttributes); var dossierAttributeConfig = new ArrayList(); - dossierAttributeConfig.add(DossierAttributeConfig.builder().type(DossierAttributeType.TEXT) - .placeholder("{{dossier.attribute.TestDossierAttribute}}").dossierTemplateId("dossierTemplateId").id("testDossierAttribute").build()); + dossierAttributeConfig.add(DossierAttributeConfig.builder() + .type(DossierAttributeType.TEXT) + .placeholder("{{dossier.attribute.TestDossierAttribute}}") + .dossierTemplateId("dossierTemplateId") + .id("testDossierAttribute") + .build()); when(dossierAttributesConfigClient.getDossierAttributes("dossierTemplateId")).thenReturn(dossierAttributeConfig); var fileAttributeConfig = new ArrayList(); - fileAttributeConfig.add(FileAttributeConfig.builder().type(FileAttributeType.TEXT) - .placeholder("{{file.attribute.TestFileAttribute}}").dossierTemplateId("dossierTemplateId").id("testFileAttribute").build()); + fileAttributeConfig.add(FileAttributeConfig.builder() + .type(FileAttributeType.TEXT) + .placeholder("{{file.attribute.TestFileAttribute}}") + .dossierTemplateId("dossierTemplateId") + .id("testFileAttribute") + .build()); when(fileAttributesConfigClient.getFileAttributeConfigs("dossierTemplateId")).thenReturn(fileAttributeConfig); - var fileModel = FileModel.builder().filename("filename").build(); - fileModel.setFileAttributes(Map.of("testFileAttribute", "File Attribute Value")); - when(fileStatusClient.getFileStatus("dossierId", "fileId")).thenReturn(fileModel); - var redactionLog = objectMapper.readValue(new ClassPathResource("files/redactionLog.json").getInputStream(), RedactionLog.class); - - when(redactionLogClient.getRedactionLog("dossierId", "fileId", new ArrayList<>(), true, false)).thenReturn(redactionLog); - + var fileModels = createFileModels(numOfFiles); + for (int i = 1; i <= numOfFiles; i++) { + when(fileStatusClient.getFileStatus("dossierId", "fileId" + i)).thenReturn(fileModels.get(i - 1)); + when(redactionLogClient.getRedactionLog("dossierId", "fileId" + i, new ArrayList<>(), true, false)).thenReturn(redactionLog); + } var templateIds = new HashSet(); for (var template : templates) { @@ -143,13 +152,14 @@ public class RedactionReportV2IntegrationTest { ClassPathResource templateResource = new ClassPathResource(template); fileSystemBackedStorageService.storeObject(templateId + "Storage", templateResource.getInputStream()); - } + var fileIds = new ArrayList(); + fileModels.forEach(fileModel -> fileIds.add(fileModel.getId())); var request = new ReportRequestMessage(); request.setDossierId("dossierId"); - request.setFileIds(List.of("fileId")); + request.setFileIds(fileIds); request.setDossierTemplateId("dossierTemplateId"); request.setDownloadId("downloadId"); request.setUserId("userId"); @@ -158,44 +168,58 @@ public class RedactionReportV2IntegrationTest { return request; } + @SneakyThrows - private void processRequest(ReportRequestMessage reportRequestMessage) { + private void processRequest(ReportRequestMessage reportRequestMessage, String fileEnding) { + var id = reportGenerationService.generateReports(reportRequestMessage); var object = fileSystemBackedStorageService.getObject(id); var infoList = objectMapper.readValue(object.getInputStream(), new TypeReference>() { }); - int i = 1; for (var fileInfo : infoList) { var report = fileSystemBackedStorageService.getObject(fileInfo.getStorageId()); byte[] reportInfo = IOUtils.toByteArray(report.getInputStream()); - try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/output-" + fileInfo.getTemplateId() + ".xlsx")) { + var outputPath = getTemporaryDirectory() + "/output-" + fileInfo.getTemplateId() + "_" + i + fileEnding; + try (FileOutputStream fileOutputStream = new FileOutputStream(outputPath)) { fileOutputStream.write(reportInfo); } - System.out.println("Created temporary output file: " + getTemporaryDirectory() + "/output-" + fileInfo.getTemplateId() + ".xlsx"); + System.out.println("Created temporary output file: " + outputPath); i++; } } + @Test @SneakyThrows public void testBasicExcelReportFlow() { - var reportRequestMessage = prepareFlow("templates/Excel Report.xlsx"); - processRequest(reportRequestMessage); + var reportRequestMessage = prepareFlow(1,"templates/Excel Report.xlsx"); + processRequest(reportRequestMessage, ".xlsx"); } + + @Test + @SneakyThrows + public void testMultifileWordReportFlow() { + + var reportRequestMessage = prepareFlow(2,"templates/Justification Appendix A1.docx"); + processRequest(reportRequestMessage, ".docx"); + } + + @Test @SneakyThrows public void test2FilesConcurrently() { - var reportRequestMessage = prepareFlow("templates/report.xlsx", "templates/report-advanced.xlsx"); - processRequest(reportRequestMessage); + var reportRequestMessage = prepareFlow(1,"templates/report.xlsx", "templates/report-advanced.xlsx"); + processRequest(reportRequestMessage, ".xlsx"); } + @Configuration @EnableAutoConfiguration(exclude = {StorageAutoConfiguration.class, RabbitAutoConfiguration.class}) @ComponentScan("com.iqser.red.service.persistence") @@ -204,10 +228,24 @@ public class RedactionReportV2IntegrationTest { @Bean @Primary public StorageService inmemoryStorage() { + return new FileSystemBackedStorageService(); } + } + + private List createFileModels(int numOfFileModelsToCreate) { + + if (numOfFileModelsToCreate <= 0) { + numOfFileModelsToCreate = 1; + } + + var fileModels = new ArrayList(); + for (int i = 1; i <= numOfFileModelsToCreate; i++) { + fileModels.add(FileModel.builder().id("fileId" + i).filename("filename" + i).fileAttributes(Map.of("testFileAttribute" + i, "File Attribute Value" + i)).build()); + } + return fileModels; } }