diff --git a/redaction-report-service-v1/redaction-report-service-api-v1/src/main/java/com/iqser/red/service/redaction/report/v1/api/model/StoredFileInformation.java b/redaction-report-service-v1/redaction-report-service-api-v1/src/main/java/com/iqser/red/service/redaction/report/v1/api/model/StoredFileInformation.java index 2f84ad8..7c66631 100644 --- a/redaction-report-service-v1/redaction-report-service-api-v1/src/main/java/com/iqser/red/service/redaction/report/v1/api/model/StoredFileInformation.java +++ b/redaction-report-service-v1/redaction-report-service-api-v1/src/main/java/com/iqser/red/service/redaction/report/v1/api/model/StoredFileInformation.java @@ -17,5 +17,6 @@ public class StoredFileInformation { private String storageId; private ReportType reportType; private String templateId; + private int documentPartNr; } diff --git a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/Application.java b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/Application.java index 6957e78..6f33fa7 100644 --- a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/Application.java +++ b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/Application.java @@ -3,10 +3,13 @@ package com.iqser.red.service.redaction.report.v1.server; import com.iqser.red.commons.spring.DefaultWebMvcConfiguration; import com.iqser.red.service.redaction.report.v1.server.client.DossierClient; import com.iqser.red.service.redaction.report.v1.server.configuration.MessagingConfiguration; +import com.iqser.red.service.redaction.report.v1.server.settings.ReportTemplateSettings; + import org.springframework.boot.SpringApplication; import org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration; +import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.cloud.openfeign.EnableFeignClients; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Import; @@ -19,6 +22,7 @@ import io.micrometer.core.instrument.MeterRegistry; @SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class}) @Import({DefaultWebMvcConfiguration.class, MessagingConfiguration.class}) @EnableFeignClients(basePackageClasses = {DossierClient.class}) +@EnableConfigurationProperties(ReportTemplateSettings.class) public class Application { /** diff --git a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/model/MultiFileDocument.java b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/model/MultiFileDocument.java index 96d4341..8a632f4 100644 --- a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/model/MultiFileDocument.java +++ b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/model/MultiFileDocument.java @@ -8,8 +8,11 @@ import org.apache.poi.xwpf.usermodel.XWPFDocument; @AllArgsConstructor public class MultiFileDocument { + private byte[] templateAsBytes; private XWPFDocument document; private String templateId; private String templateName; + private int numberOfEntries; + private int documentPartNr; } diff --git a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ReportGenerationService.java b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ReportGenerationService.java index efed4d8..180957a 100644 --- a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ReportGenerationService.java +++ b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ReportGenerationService.java @@ -31,6 +31,7 @@ import com.iqser.red.service.redaction.report.v1.server.model.MultiFileDocument; import com.iqser.red.service.redaction.report.v1.server.model.MultiFileWorkbook; import com.iqser.red.service.redaction.report.v1.server.model.PlaceholderModel; import com.iqser.red.service.redaction.report.v1.server.model.ReportRedactionEntry; +import com.iqser.red.service.redaction.report.v1.server.settings.ReportTemplateSettings; import com.iqser.red.service.redaction.report.v1.server.storage.ReportStorageService; import com.iqser.red.service.redaction.v1.model.RedactionLog; import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist; @@ -56,7 +57,7 @@ public class ReportGenerationService { private final ExcelReportGenerationService excelTemplateReportGenerationService; private final GeneratePlaceholderService generatePlaceholderService; private final DictionaryClient dictionaryClient; - + private final ReportTemplateSettings reportTemplateSettings; @SneakyThrows @Timed("redactmanager_generateReports") @@ -90,7 +91,7 @@ public class ReportGenerationService { byte[] wordTemplate = reportStorageService.getReportTemplate(reportTemplate.getStorageId()); try (ByteArrayInputStream is = new ByteArrayInputStream(wordTemplate)) { XWPFDocument doc = new XWPFDocument(is); - MultiFileDocument multiFileDocument = new MultiFileDocument(doc, templateId, reportTemplate.getFileName()); + MultiFileDocument multiFileDocument = new MultiFileDocument(wordTemplate, doc, templateId, reportTemplate.getFileName(), 0, 0); multiFileDocuments.add(multiFileDocument); } catch (IOException e) { throw new RuntimeException("Could not generate multifile word report."); @@ -112,24 +113,53 @@ public class ReportGenerationService { long start = System.currentTimeMillis(); - var fileStatus = fileStatusClient.getFileStatus(reportMessage.getDossierId(), reportMessage.getFileIds().get(j)); + var fileStatus = fileStatusClient.getFileStatus(reportMessage.getDossierId(), reportMessage.getFileIds() + .get(j)); generatePlaceholderService.resolveFileAttributeValues(fileStatus, placeholderModel); - List reportEntries = getReportEntries(reportMessage.getDossierId(), reportMessage.getFileIds().get(j), fileStatus.isExcluded()); + List reportEntries = getReportEntries(reportMessage.getDossierId(), reportMessage.getFileIds() + .get(j), fileStatus.isExcluded()); var isLastFile = j == reportMessage.getFileIds().size() - 1; for (MultiFileWorkbook multiFileWorkbook : multiFileWorkbooks) { if (excelModel == null) { - excelModel = excelTemplateReportGenerationService.calculateExcelModel(multiFileWorkbook.getReadWorkBook().getSheetAt(0)); + excelModel = excelTemplateReportGenerationService.calculateExcelModel(multiFileWorkbook.getReadWorkBook() + .getSheetAt(0)); } excelTemplateReportGenerationService.generateExcelReport(reportEntries, placeholderModel, multiFileWorkbook.getTemplateName(), multiFileWorkbook.getWriteWorkbook(), dossier.getDossierName(), fileStatus, excelModel, isLastFile); } for (MultiFileDocument multiFileDocument : multiFileDocuments) { - wordReportGenerationService.generateWordReport(reportEntries, placeholderModel, multiFileDocument.getTemplateName(), multiFileDocument.getDocument(), fileStatus, dossier, isLastFile); + if (reportEntries.size() >= reportTemplateSettings.getMultiFileChunkSize()) { + + try (ByteArrayInputStream is = new ByteArrayInputStream(multiFileDocument.getTemplateAsBytes())) { + XWPFDocument doc = new XWPFDocument(is); + wordReportGenerationService.generateWordReport(reportEntries, placeholderModel, multiFileDocument.getTemplateName(), doc, fileStatus, dossier, true); + byte[] wordDoc = wordReportGenerationService.toByteArray(doc); + String storageId = reportStorageService.storeObject(reportMessage.getDownloadId(), wordDoc); + storedFileInformation.add(new StoredFileInformation(null, storageId, ReportType.WORD_TEMPLATE_MULTI_FILE, multiFileDocument.getTemplateId(), multiFileDocument.getDocumentPartNr())); + multiFileDocument.setDocumentPartNr(multiFileDocument.getDocumentPartNr() + 1); + } + continue; + } + + if (multiFileDocument.getNumberOfEntries() >= reportTemplateSettings.getMultiFileChunkSize()) { + byte[] wordDoc = wordReportGenerationService.toByteArray(multiFileDocument.getDocument()); + String storageId = reportStorageService.storeObject(reportMessage.getDownloadId(), wordDoc); + storedFileInformation.add(new StoredFileInformation(null, storageId, ReportType.WORD_TEMPLATE_MULTI_FILE, multiFileDocument.getTemplateId(), multiFileDocument.getDocumentPartNr())); + multiFileDocument.setDocumentPartNr(multiFileDocument.getDocumentPartNr() + 1); + + try (ByteArrayInputStream is = new ByteArrayInputStream(multiFileDocument.getTemplateAsBytes())) { + XWPFDocument doc = new XWPFDocument(is); + multiFileDocument.setDocument(doc); + multiFileDocument.setNumberOfEntries(0); + } + } + wordReportGenerationService.generateWordReport(reportEntries, placeholderModel, multiFileDocument.getTemplateName(), multiFileDocument.getDocument(), fileStatus, dossier, isLastFile ? isLastFile : multiFileDocument.getNumberOfEntries() + reportEntries.size() >= reportTemplateSettings.getMultiFileChunkSize()); + multiFileDocument.setNumberOfEntries(multiFileDocument.getNumberOfEntries() + reportEntries.size()); } for (ReportTemplate reportTemplate : singleFilesTemplates) { @@ -138,28 +168,31 @@ public class ReportGenerationService { } long end = System.currentTimeMillis(); - log.info("Successfully processed {}/{} fileIds for downloadId {}, took {}", i, reportMessage.getFileIds().size(), reportMessage.getDownloadId(), end - start); + log.info("Successfully processed {}/{} fileIds for downloadId {}, took {}", i, reportMessage.getFileIds() + .size(), reportMessage.getDownloadId(), end - start); i++; } for (MultiFileWorkbook multiFileWorkbook : multiFileWorkbooks) { byte[] template = excelTemplateReportGenerationService.toByteArray(multiFileWorkbook.getWriteWorkbook()); String storageId = reportStorageService.storeObject(reportMessage.getDownloadId(), template); - storedFileInformation.add(new StoredFileInformation(null, storageId, ReportType.EXCEL_TEMPLATE_MULTI_FILE, multiFileWorkbook.getTemplateId())); + storedFileInformation.add(new StoredFileInformation(null, storageId, ReportType.EXCEL_TEMPLATE_MULTI_FILE, multiFileWorkbook.getTemplateId(), 0)); } for (MultiFileDocument multiFileDocument : multiFileDocuments) { byte[] template = wordReportGenerationService.toByteArray(multiFileDocument.getDocument()); String storageId = reportStorageService.storeObject(reportMessage.getDownloadId(), template); - storedFileInformation.add(new StoredFileInformation(null, storageId, ReportType.WORD_TEMPLATE_MULTI_FILE, multiFileDocument.getTemplateId())); + storedFileInformation.add(new StoredFileInformation(null, storageId, ReportType.WORD_TEMPLATE_MULTI_FILE, multiFileDocument.getTemplateId(), multiFileDocument.getDocumentPartNr())); } return reportStorageService.storeReportInformation(reportMessage.getDownloadId(), storedFileInformation); } - private StoredFileInformation createReportFromTemplate(Dossier dossier, FileModel fileStatus, PlaceholderModel placeholderModel, String templateName, String downloadId, - List reportEntries, ReportTemplate reportTemplate) { + private StoredFileInformation createReportFromTemplate(Dossier dossier, FileModel fileStatus, + PlaceholderModel placeholderModel, String templateName, + String downloadId, List reportEntries, + ReportTemplate reportTemplate) { if (reportTemplate.getFileName().endsWith(".xlsx")) { byte[] excelTemplate = reportStorageService.getReportTemplate(reportTemplate.getStorageId()); @@ -173,7 +206,7 @@ public class ReportGenerationService { excelTemplateReportGenerationService.generateExcelReport(reportEntries, placeholderModel, templateName, writeWorkbook, dossier.getDossierName(), fileStatus, excelModel, true); byte[] template = excelTemplateReportGenerationService.toByteArray(writeWorkbook); String storageId = reportStorageService.storeObject(downloadId, template); - return new StoredFileInformation(fileStatus.getId(), storageId, ReportType.EXCEL_TEMPLATE_SINGLE_FILE, reportTemplate.getTemplateId()); + return new StoredFileInformation(fileStatus.getId(), storageId, ReportType.EXCEL_TEMPLATE_SINGLE_FILE, reportTemplate.getTemplateId(), 0); } catch (IOException e) { throw new RuntimeException("Could not generate singlefile excel report."); } @@ -184,7 +217,7 @@ public class ReportGenerationService { wordReportGenerationService.generateWordReport(reportEntries, placeholderModel, templateName, doc, fileStatus, dossier, true); byte[] template = wordReportGenerationService.toByteArray(doc); String storageId = reportStorageService.storeObject(downloadId, template); - return new StoredFileInformation(fileStatus.getId(), storageId, ReportType.WORD_SINGLE_FILE, reportTemplate.getTemplateId()); + return new StoredFileInformation(fileStatus.getId(), storageId, ReportType.WORD_SINGLE_FILE, reportTemplate.getTemplateId(), 0); } catch (IOException e) { throw new RuntimeException("Could not generate singlefile word report."); } @@ -216,8 +249,9 @@ public class ReportGenerationService { private Map fillMapOfTypeAndEntityDisplayName(String dossierId) { List typeList = new ArrayList<>(); - typeList.addAll(dictionaryClient.getAllTypesForDossier(dossierId,false)); - typeList.addAll(dictionaryClient.getAllTypesForDossierTemplate(dossierClient.getDossierById(dossierId, true, false).getDossierTemplateId(), false)); + typeList.addAll(dictionaryClient.getAllTypesForDossier(dossierId, false)); + typeList.addAll(dictionaryClient.getAllTypesForDossierTemplate(dossierClient.getDossierById(dossierId, true, false) + .getDossierTemplateId(), false)); Map mapOfEntityDisplayName = new HashMap<>(); diff --git a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/settings/ReportTemplateSettings.java b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/settings/ReportTemplateSettings.java new file mode 100644 index 0000000..6b6d8e2 --- /dev/null +++ b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/settings/ReportTemplateSettings.java @@ -0,0 +1,13 @@ +package com.iqser.red.service.redaction.report.v1.server.settings; + +import org.springframework.boot.context.properties.ConfigurationProperties; + +import lombok.Data; + +@Data +@ConfigurationProperties("redaction-report-service") +public class ReportTemplateSettings { + + private int multiFileChunkSize = 2000; + +} \ No newline at end of file