From bb616e4a19654dbcb47b0069f174390f6f113e74 Mon Sep 17 00:00:00 2001 From: deiflaender Date: Mon, 8 Aug 2022 11:26:04 +0200 Subject: [PATCH 1/2] RED-4610: Enabled chunked word multifiles --- .../v1/api/model/StoredFileInformation.java | 1 + .../v1/server/model/MultiFileDocument.java | 3 + .../service/ReportGenerationService.java | 61 ++++++++++++++----- 3 files changed, 51 insertions(+), 14 deletions(-) diff --git a/redaction-report-service-v1/redaction-report-service-api-v1/src/main/java/com/iqser/red/service/redaction/report/v1/api/model/StoredFileInformation.java b/redaction-report-service-v1/redaction-report-service-api-v1/src/main/java/com/iqser/red/service/redaction/report/v1/api/model/StoredFileInformation.java index 2f84ad8..7c66631 100644 --- a/redaction-report-service-v1/redaction-report-service-api-v1/src/main/java/com/iqser/red/service/redaction/report/v1/api/model/StoredFileInformation.java +++ b/redaction-report-service-v1/redaction-report-service-api-v1/src/main/java/com/iqser/red/service/redaction/report/v1/api/model/StoredFileInformation.java @@ -17,5 +17,6 @@ public class StoredFileInformation { private String storageId; private ReportType reportType; private String templateId; + private int documentPartNr; } diff --git a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/model/MultiFileDocument.java b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/model/MultiFileDocument.java index 96d4341..8a632f4 100644 --- a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/model/MultiFileDocument.java +++ b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/model/MultiFileDocument.java @@ -8,8 +8,11 @@ import org.apache.poi.xwpf.usermodel.XWPFDocument; @AllArgsConstructor public class MultiFileDocument { + private byte[] templateAsBytes; private XWPFDocument document; private String templateId; private String templateName; + private int numberOfEntries; + private int documentPartNr; } diff --git a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ReportGenerationService.java b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ReportGenerationService.java index efed4d8..d357082 100644 --- a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ReportGenerationService.java +++ b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ReportGenerationService.java @@ -90,7 +90,7 @@ public class ReportGenerationService { byte[] wordTemplate = reportStorageService.getReportTemplate(reportTemplate.getStorageId()); try (ByteArrayInputStream is = new ByteArrayInputStream(wordTemplate)) { XWPFDocument doc = new XWPFDocument(is); - MultiFileDocument multiFileDocument = new MultiFileDocument(doc, templateId, reportTemplate.getFileName()); + MultiFileDocument multiFileDocument = new MultiFileDocument(wordTemplate, doc, templateId, reportTemplate.getFileName(), 0, 0); multiFileDocuments.add(multiFileDocument); } catch (IOException e) { throw new RuntimeException("Could not generate multifile word report."); @@ -112,24 +112,53 @@ public class ReportGenerationService { long start = System.currentTimeMillis(); - var fileStatus = fileStatusClient.getFileStatus(reportMessage.getDossierId(), reportMessage.getFileIds().get(j)); + var fileStatus = fileStatusClient.getFileStatus(reportMessage.getDossierId(), reportMessage.getFileIds() + .get(j)); generatePlaceholderService.resolveFileAttributeValues(fileStatus, placeholderModel); - List reportEntries = getReportEntries(reportMessage.getDossierId(), reportMessage.getFileIds().get(j), fileStatus.isExcluded()); + List reportEntries = getReportEntries(reportMessage.getDossierId(), reportMessage.getFileIds() + .get(j), fileStatus.isExcluded()); var isLastFile = j == reportMessage.getFileIds().size() - 1; for (MultiFileWorkbook multiFileWorkbook : multiFileWorkbooks) { if (excelModel == null) { - excelModel = excelTemplateReportGenerationService.calculateExcelModel(multiFileWorkbook.getReadWorkBook().getSheetAt(0)); + excelModel = excelTemplateReportGenerationService.calculateExcelModel(multiFileWorkbook.getReadWorkBook() + .getSheetAt(0)); } excelTemplateReportGenerationService.generateExcelReport(reportEntries, placeholderModel, multiFileWorkbook.getTemplateName(), multiFileWorkbook.getWriteWorkbook(), dossier.getDossierName(), fileStatus, excelModel, isLastFile); } for (MultiFileDocument multiFileDocument : multiFileDocuments) { - wordReportGenerationService.generateWordReport(reportEntries, placeholderModel, multiFileDocument.getTemplateName(), multiFileDocument.getDocument(), fileStatus, dossier, isLastFile); + if (reportEntries.size() >= 2000) { + + try (ByteArrayInputStream is = new ByteArrayInputStream(multiFileDocument.getTemplateAsBytes())) { + XWPFDocument doc = new XWPFDocument(is); + wordReportGenerationService.generateWordReport(reportEntries, placeholderModel, multiFileDocument.getTemplateName(), doc, fileStatus, dossier, true); + byte[] wordDoc = wordReportGenerationService.toByteArray(doc); + String storageId = reportStorageService.storeObject(reportMessage.getDownloadId(), wordDoc); + storedFileInformation.add(new StoredFileInformation(null, storageId, ReportType.WORD_TEMPLATE_MULTI_FILE, multiFileDocument.getTemplateId(), multiFileDocument.getDocumentPartNr())); + multiFileDocument.setDocumentPartNr(multiFileDocument.getDocumentPartNr() + 1); + } + continue; + } + + if (multiFileDocument.getNumberOfEntries() >= 2000) { + byte[] wordDoc = wordReportGenerationService.toByteArray(multiFileDocument.getDocument()); + String storageId = reportStorageService.storeObject(reportMessage.getDownloadId(), wordDoc); + storedFileInformation.add(new StoredFileInformation(null, storageId, ReportType.WORD_TEMPLATE_MULTI_FILE, multiFileDocument.getTemplateId(), multiFileDocument.getDocumentPartNr())); + multiFileDocument.setDocumentPartNr(multiFileDocument.getDocumentPartNr() + 1); + + try (ByteArrayInputStream is = new ByteArrayInputStream(multiFileDocument.getTemplateAsBytes())) { + XWPFDocument doc = new XWPFDocument(is); + multiFileDocument.setDocument(doc); + multiFileDocument.setNumberOfEntries(0); + } + } + wordReportGenerationService.generateWordReport(reportEntries, placeholderModel, multiFileDocument.getTemplateName(), multiFileDocument.getDocument(), fileStatus, dossier, isLastFile ? true : multiFileDocument.getNumberOfEntries() + reportEntries.size() >= 2000); + multiFileDocument.setNumberOfEntries(multiFileDocument.getNumberOfEntries() + reportEntries.size()); } for (ReportTemplate reportTemplate : singleFilesTemplates) { @@ -138,28 +167,31 @@ public class ReportGenerationService { } long end = System.currentTimeMillis(); - log.info("Successfully processed {}/{} fileIds for downloadId {}, took {}", i, reportMessage.getFileIds().size(), reportMessage.getDownloadId(), end - start); + log.info("Successfully processed {}/{} fileIds for downloadId {}, took {}", i, reportMessage.getFileIds() + .size(), reportMessage.getDownloadId(), end - start); i++; } for (MultiFileWorkbook multiFileWorkbook : multiFileWorkbooks) { byte[] template = excelTemplateReportGenerationService.toByteArray(multiFileWorkbook.getWriteWorkbook()); String storageId = reportStorageService.storeObject(reportMessage.getDownloadId(), template); - storedFileInformation.add(new StoredFileInformation(null, storageId, ReportType.EXCEL_TEMPLATE_MULTI_FILE, multiFileWorkbook.getTemplateId())); + storedFileInformation.add(new StoredFileInformation(null, storageId, ReportType.EXCEL_TEMPLATE_MULTI_FILE, multiFileWorkbook.getTemplateId(), 0)); } for (MultiFileDocument multiFileDocument : multiFileDocuments) { byte[] template = wordReportGenerationService.toByteArray(multiFileDocument.getDocument()); String storageId = reportStorageService.storeObject(reportMessage.getDownloadId(), template); - storedFileInformation.add(new StoredFileInformation(null, storageId, ReportType.WORD_TEMPLATE_MULTI_FILE, multiFileDocument.getTemplateId())); + storedFileInformation.add(new StoredFileInformation(null, storageId, ReportType.WORD_TEMPLATE_MULTI_FILE, multiFileDocument.getTemplateId(), multiFileDocument.getDocumentPartNr())); } return reportStorageService.storeReportInformation(reportMessage.getDownloadId(), storedFileInformation); } - private StoredFileInformation createReportFromTemplate(Dossier dossier, FileModel fileStatus, PlaceholderModel placeholderModel, String templateName, String downloadId, - List reportEntries, ReportTemplate reportTemplate) { + private StoredFileInformation createReportFromTemplate(Dossier dossier, FileModel fileStatus, + PlaceholderModel placeholderModel, String templateName, + String downloadId, List reportEntries, + ReportTemplate reportTemplate) { if (reportTemplate.getFileName().endsWith(".xlsx")) { byte[] excelTemplate = reportStorageService.getReportTemplate(reportTemplate.getStorageId()); @@ -173,7 +205,7 @@ public class ReportGenerationService { excelTemplateReportGenerationService.generateExcelReport(reportEntries, placeholderModel, templateName, writeWorkbook, dossier.getDossierName(), fileStatus, excelModel, true); byte[] template = excelTemplateReportGenerationService.toByteArray(writeWorkbook); String storageId = reportStorageService.storeObject(downloadId, template); - return new StoredFileInformation(fileStatus.getId(), storageId, ReportType.EXCEL_TEMPLATE_SINGLE_FILE, reportTemplate.getTemplateId()); + return new StoredFileInformation(fileStatus.getId(), storageId, ReportType.EXCEL_TEMPLATE_SINGLE_FILE, reportTemplate.getTemplateId(), 0); } catch (IOException e) { throw new RuntimeException("Could not generate singlefile excel report."); } @@ -184,7 +216,7 @@ public class ReportGenerationService { wordReportGenerationService.generateWordReport(reportEntries, placeholderModel, templateName, doc, fileStatus, dossier, true); byte[] template = wordReportGenerationService.toByteArray(doc); String storageId = reportStorageService.storeObject(downloadId, template); - return new StoredFileInformation(fileStatus.getId(), storageId, ReportType.WORD_SINGLE_FILE, reportTemplate.getTemplateId()); + return new StoredFileInformation(fileStatus.getId(), storageId, ReportType.WORD_SINGLE_FILE, reportTemplate.getTemplateId(), 0); } catch (IOException e) { throw new RuntimeException("Could not generate singlefile word report."); } @@ -216,8 +248,9 @@ public class ReportGenerationService { private Map fillMapOfTypeAndEntityDisplayName(String dossierId) { List typeList = new ArrayList<>(); - typeList.addAll(dictionaryClient.getAllTypesForDossier(dossierId,false)); - typeList.addAll(dictionaryClient.getAllTypesForDossierTemplate(dossierClient.getDossierById(dossierId, true, false).getDossierTemplateId(), false)); + typeList.addAll(dictionaryClient.getAllTypesForDossier(dossierId, false)); + typeList.addAll(dictionaryClient.getAllTypesForDossierTemplate(dossierClient.getDossierById(dossierId, true, false) + .getDossierTemplateId(), false)); Map mapOfEntityDisplayName = new HashMap<>(); From 6b780873757ea9efdb6ff94a93fe0acf0c2f389f Mon Sep 17 00:00:00 2001 From: Ali Oezyetimoglu Date: Mon, 8 Aug 2022 15:49:24 +0200 Subject: [PATCH 2/2] RED-4610: Support of multi-file word report templates --- .../redaction/report/v1/server/Application.java | 4 ++++ .../v1/server/service/ReportGenerationService.java | 9 +++++---- .../v1/server/settings/ReportTemplateSettings.java | 13 +++++++++++++ 3 files changed, 22 insertions(+), 4 deletions(-) create mode 100644 redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/settings/ReportTemplateSettings.java diff --git a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/Application.java b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/Application.java index 6957e78..6f33fa7 100644 --- a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/Application.java +++ b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/Application.java @@ -3,10 +3,13 @@ package com.iqser.red.service.redaction.report.v1.server; import com.iqser.red.commons.spring.DefaultWebMvcConfiguration; import com.iqser.red.service.redaction.report.v1.server.client.DossierClient; import com.iqser.red.service.redaction.report.v1.server.configuration.MessagingConfiguration; +import com.iqser.red.service.redaction.report.v1.server.settings.ReportTemplateSettings; + import org.springframework.boot.SpringApplication; import org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration; +import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.cloud.openfeign.EnableFeignClients; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Import; @@ -19,6 +22,7 @@ import io.micrometer.core.instrument.MeterRegistry; @SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class}) @Import({DefaultWebMvcConfiguration.class, MessagingConfiguration.class}) @EnableFeignClients(basePackageClasses = {DossierClient.class}) +@EnableConfigurationProperties(ReportTemplateSettings.class) public class Application { /** diff --git a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ReportGenerationService.java b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ReportGenerationService.java index d357082..180957a 100644 --- a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ReportGenerationService.java +++ b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ReportGenerationService.java @@ -31,6 +31,7 @@ import com.iqser.red.service.redaction.report.v1.server.model.MultiFileDocument; import com.iqser.red.service.redaction.report.v1.server.model.MultiFileWorkbook; import com.iqser.red.service.redaction.report.v1.server.model.PlaceholderModel; import com.iqser.red.service.redaction.report.v1.server.model.ReportRedactionEntry; +import com.iqser.red.service.redaction.report.v1.server.settings.ReportTemplateSettings; import com.iqser.red.service.redaction.report.v1.server.storage.ReportStorageService; import com.iqser.red.service.redaction.v1.model.RedactionLog; import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist; @@ -56,7 +57,7 @@ public class ReportGenerationService { private final ExcelReportGenerationService excelTemplateReportGenerationService; private final GeneratePlaceholderService generatePlaceholderService; private final DictionaryClient dictionaryClient; - + private final ReportTemplateSettings reportTemplateSettings; @SneakyThrows @Timed("redactmanager_generateReports") @@ -132,7 +133,7 @@ public class ReportGenerationService { for (MultiFileDocument multiFileDocument : multiFileDocuments) { - if (reportEntries.size() >= 2000) { + if (reportEntries.size() >= reportTemplateSettings.getMultiFileChunkSize()) { try (ByteArrayInputStream is = new ByteArrayInputStream(multiFileDocument.getTemplateAsBytes())) { XWPFDocument doc = new XWPFDocument(is); @@ -145,7 +146,7 @@ public class ReportGenerationService { continue; } - if (multiFileDocument.getNumberOfEntries() >= 2000) { + if (multiFileDocument.getNumberOfEntries() >= reportTemplateSettings.getMultiFileChunkSize()) { byte[] wordDoc = wordReportGenerationService.toByteArray(multiFileDocument.getDocument()); String storageId = reportStorageService.storeObject(reportMessage.getDownloadId(), wordDoc); storedFileInformation.add(new StoredFileInformation(null, storageId, ReportType.WORD_TEMPLATE_MULTI_FILE, multiFileDocument.getTemplateId(), multiFileDocument.getDocumentPartNr())); @@ -157,7 +158,7 @@ public class ReportGenerationService { multiFileDocument.setNumberOfEntries(0); } } - wordReportGenerationService.generateWordReport(reportEntries, placeholderModel, multiFileDocument.getTemplateName(), multiFileDocument.getDocument(), fileStatus, dossier, isLastFile ? true : multiFileDocument.getNumberOfEntries() + reportEntries.size() >= 2000); + wordReportGenerationService.generateWordReport(reportEntries, placeholderModel, multiFileDocument.getTemplateName(), multiFileDocument.getDocument(), fileStatus, dossier, isLastFile ? isLastFile : multiFileDocument.getNumberOfEntries() + reportEntries.size() >= reportTemplateSettings.getMultiFileChunkSize()); multiFileDocument.setNumberOfEntries(multiFileDocument.getNumberOfEntries() + reportEntries.size()); } diff --git a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/settings/ReportTemplateSettings.java b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/settings/ReportTemplateSettings.java new file mode 100644 index 0000000..6b6d8e2 --- /dev/null +++ b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/settings/ReportTemplateSettings.java @@ -0,0 +1,13 @@ +package com.iqser.red.service.redaction.report.v1.server.settings; + +import org.springframework.boot.context.properties.ConfigurationProperties; + +import lombok.Data; + +@Data +@ConfigurationProperties("redaction-report-service") +public class ReportTemplateSettings { + + private int multiFileChunkSize = 2000; + +} \ No newline at end of file