RED-4036: Reenabled all replacememts in excel

This commit is contained in:
deiflaender 2022-05-20 14:46:13 +02:00
parent f49ece3add
commit 93fcc061f9
12 changed files with 245 additions and 199 deletions

View File

@ -54,11 +54,6 @@
<version>5.0.0</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-aop</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-amqp</artifactId>

View File

@ -0,0 +1,13 @@
package com.iqser.red.service.redaction.report.v1.server.model;
import org.apache.poi.ss.usermodel.Cell;
import lombok.AllArgsConstructor;
@AllArgsConstructor
public class ExcelCell {
private Cell cell;
private int columnWidth;
private int rowHeights;
}

View File

@ -1,6 +1,8 @@
package com.iqser.red.service.redaction.report.v1.server.model;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
@ -16,9 +18,12 @@ import lombok.NoArgsConstructor;
public class ExcelModel {
private Map<Integer, Function<PlaceholderInput, String>> placeholderCellPos;
private int placeholderRow;
private int redactionPlaceholderRow;
private Map<Integer, Integer> cellWidths = new HashMap<>();
private Map<CellIdentifier, Cell> cellsToCopy = new HashMap<>();
private boolean copyDone;
private Map<CellIdentifier, Cell> cellsToCopyBeforeRedactionPlaceholderRow = new HashMap<>();
private Map<CellIdentifier, Cell> cellsToCopyAfterRedactionPlaceholderRow = new HashMap<>();
private List<Integer> writtenRows = new ArrayList<>();
private boolean rowsBeforeRedactionEntryRowsAdded;
}

View File

@ -25,6 +25,7 @@ import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@ -37,11 +38,6 @@ import java.util.regex.Pattern;
import javax.imageio.ImageIO;
import com.iqser.red.service.redaction.report.v1.server.model.CellIdentifier;
import com.iqser.red.service.redaction.report.v1.server.model.ExcelModel;
import com.iqser.red.service.redaction.report.v1.server.model.ImagePlaceholder;
import com.iqser.red.service.redaction.report.v1.server.model.PlaceholderModel;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.CellStyle;
import org.apache.poi.ss.usermodel.ClientAnchor;
@ -51,14 +47,17 @@ import org.apache.poi.ss.usermodel.Picture;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.Dossier;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel;
import com.iqser.red.service.redaction.report.v1.server.model.ReportRedactionEntry;
import com.iqser.red.service.redaction.report.v1.server.model.CellIdentifier;
import com.iqser.red.service.redaction.report.v1.server.model.ExcelModel;
import com.iqser.red.service.redaction.report.v1.server.model.ImagePlaceholder;
import com.iqser.red.service.redaction.report.v1.server.model.PlaceholderInput;
import com.iqser.red.service.redaction.report.v1.server.model.PlaceholderModel;
import com.iqser.red.service.redaction.report.v1.server.model.ReportRedactionEntry;
import io.micrometer.core.annotation.Timed;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
@ -66,71 +65,94 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@RequiredArgsConstructor
@SuppressWarnings("PMD")
public class ExcelTemplateReportGenerationService {
public class ExcelReportGenerationService {
@SuppressWarnings("checkstyle:ParameterAssignment")
public ExcelModel generateReport(List<ReportRedactionEntry> reportEntries, PlaceholderModel placeholderModel,
String reportTemplateName, SXSSFWorkbook workbook, FileModel fileStatus,
ExcelModel excelModel) {
@Timed("generateExcelReport")
public void generateExcelReport(List<ReportRedactionEntry> reportEntries, PlaceholderModel placeholderModel,
String reportTemplateName, SXSSFWorkbook workbook, String dossierName,
FileModel fileModel, ExcelModel excelModel, boolean isLastFile) {
long start = System.currentTimeMillis();
try {
for (Sheet sheet : workbook) {
if(!excelModel.isCopyDone()) {
if (!excelModel.isRowsBeforeRedactionEntryRowsAdded()) {
for (Map.Entry<Integer, Integer> colummWidthEntry : excelModel.getCellWidths().entrySet()) {
sheet.setColumnWidth(colummWidthEntry.getKey(), colummWidthEntry.getValue());
}
Set<Integer> createdRows = new HashSet<>();
for (Map.Entry<CellIdentifier, Cell> cellsToCopyEntry : excelModel.getCellsToCopy().entrySet()) {
if (!createdRows.contains(cellsToCopyEntry.getKey().getRowIndex())) {
sheet.createRow(cellsToCopyEntry.getKey().getRowIndex());
createdRows.add(cellsToCopyEntry.getKey().getRowIndex());
}
var createdCell = sheet.getRow(cellsToCopyEntry.getKey().getRowIndex())
.createCell(cellsToCopyEntry.getKey().getColumnIndex());
createdCell.setCellValue(cellsToCopyEntry.getValue().getStringCellValue());
CellStyle newCellStyle = workbook.createCellStyle();
newCellStyle.cloneStyleFrom(cellsToCopyEntry.getValue().getCellStyle());
createdCell.setCellStyle(newCellStyle);
}
excelModel.setCopyDone(true);
addRows(workbook, sheet, excelModel.getCellsToCopyBeforeRedactionPlaceholderRow(), 0, placeholderModel, dossierName, fileModel.getFilename());
excelModel.setRowsBeforeRedactionEntryRowsAdded(true);
}
excelModel = addEntryRows(sheet, reportEntries, fileStatus.getFilename(), excelModel, placeholderModel);
addRedactionEntryRows(sheet, reportEntries, fileModel.getFilename(), excelModel, placeholderModel);
// for (ImagePlaceholder imagePlaceholder : placeholderModel.getImagePlaceholders()){
// replacePlaceholdersForImagePlaceholder(workbook, sheet, imagePlaceholder);
// }
// for (String placeholder : placeholderModel.getPlaceholders()) {
// String placeholderValue = getPlaceholderValue(placeholder, dossier, fileStatus, placeholderModel.getFileAttributePlaceHolders(), placeholderModel.getDossierAttributesPlaceholder());
// if (placeholderValue != null) {
// replacePlaceholdersForTextPlaceholder(sheet, placeholder, placeholderValue);
// }
// }
if (isLastFile) {
addRows(workbook, sheet, excelModel.getCellsToCopyAfterRedactionPlaceholderRow(), excelModel.getWrittenRows()
.size(), placeholderModel, dossierName, fileModel.getFilename());
}
}
long end = System.currentTimeMillis();
log.info("Report Generation took: {} for file with id {}, pageCount: {}, redactionLogEntryCount: {}, reportName: {}, className: {}", end - start, fileStatus.getId(), fileStatus.getNumberOfPages(), reportEntries.size(), reportTemplateName, getClass().getSimpleName());
log.info("Report Generation took: {} for file with id {}, pageCount: {}, redactionLogEntryCount: {}, reportName: {}, className: {}", System.currentTimeMillis() - start, fileModel.getId(), fileModel.getNumberOfPages(), reportEntries.size(), reportTemplateName, getClass().getSimpleName());
} catch (Exception e) {
throw new RuntimeException(e);
}
return excelModel;
}
public ExcelModel caculateExcelModel(Sheet sheet) {
@SneakyThrows
@Timed("excel-addRows")
private void addRows(SXSSFWorkbook workbook, Sheet sheet, Map<CellIdentifier, Cell> copiedCells,
int numberOfRowsToShift, PlaceholderModel placeholderModel, String dossierName,
String filename) {
Set<Integer> createdCopyRows = new HashSet<>();
for (Map.Entry<CellIdentifier, Cell> cellsToCopyEntry : copiedCells.entrySet()) {
int indexToAddRow = cellsToCopyEntry.getKey().getRowIndex() + numberOfRowsToShift;
if (!createdCopyRows.contains(indexToAddRow)) {
sheet.createRow(indexToAddRow);
createdCopyRows.add(indexToAddRow);
}
var createdCell = sheet.getRow(indexToAddRow).createCell(cellsToCopyEntry.getKey().getColumnIndex());
createdCell.setCellValue(cellsToCopyEntry.getValue().getStringCellValue());
CellStyle newCellStyle = workbook.createCellStyle();
newCellStyle.cloneStyleFrom(cellsToCopyEntry.getValue().getCellStyle());
createdCell.setCellStyle(newCellStyle);
createdCell.getRow().setHeight(cellsToCopyEntry.getValue().getRow().getHeight());
replacePlaceholders(createdCell, placeholderModel, dossierName, filename);
replacePlaceholdersForImagePlaceholder(workbook, sheet, createdCell, placeholderModel);
}
}
private void replacePlaceholders(Cell cell, PlaceholderModel placeholderModel, String dossierName,
String filename) {
for (String placeholder : placeholderModel.getPlaceholders()) {
String placeholderValue = getPlaceholderValue(placeholder, dossierName, filename, placeholderModel);
if (placeholderValue != null) {
replacePlaceholdersForCell(cell, placeholder, placeholderValue);
}
}
}
@Timed("calculateExcelModel")
public ExcelModel calculateExcelModel(Sheet sheet) {
long start = System.currentTimeMillis();
Map<Integer, Function<PlaceholderInput, String>> placeholderCellPos = new HashMap<>();
Map<Integer, Integer> columnWidths = new HashMap<>();
Map<CellIdentifier, Cell> cellsToCopy = new HashMap<>();
Map<CellIdentifier, Cell> cellsToCopyBeforePlaceholderRow = new HashMap<>();
Map<CellIdentifier, Cell> cellsToCopyAfterPlaceholderRow = new HashMap<>();
int placeholderRow = -1;
for (int j = 0; j < sheet.getLastRowNum() + 1; j++) {
Row actualRow = sheet.getRow(j);
@ -138,42 +160,51 @@ public class ExcelTemplateReportGenerationService {
for (int i = 0; i < actualRow.getLastCellNum(); i++) {
Cell cell = sheet.getRow(j).getCell(i);
if (cell != null) {
int columnWidth = sheet.getColumnWidth(i);
columnWidths.put(i, columnWidth);
if (containsRedactionPlaceholder(cell.getStringCellValue())) {
int columnWidth = sheet.getColumnWidth(i);
columnWidths.put(i, columnWidth);
placeholderCellPos.put(i, getFunctionForPlaceHolder(cell.getStringCellValue()));
placeholderRow = j;
} else {
cellsToCopy.put(new CellIdentifier(j, i), cell);
if (placeholderRow == -1) {
cellsToCopyBeforePlaceholderRow.put(new CellIdentifier(j, i), cell);
} else {
cellsToCopyAfterPlaceholderRow.put(new CellIdentifier(j, i), cell);
}
}
}
}
}
}
log.debug("Calculate Placeholder Cells took: {}", System.currentTimeMillis() - start);
return new ExcelModel(placeholderCellPos, placeholderRow, columnWidths, cellsToCopy, false);
return new ExcelModel(placeholderCellPos, placeholderRow, columnWidths, cellsToCopyBeforePlaceholderRow, cellsToCopyAfterPlaceholderRow, new ArrayList<>(), false);
}
@SuppressWarnings("checkstyle:ParameterAssignment")
private ExcelModel addEntryRows(Sheet sheet, List<ReportRedactionEntry> reportEntries, String filename, ExcelModel excelModel, PlaceholderModel placeholderModel) {
@Timed("excel-addRedactionEntryRows")
private void addRedactionEntryRows(Sheet sheet, List<ReportRedactionEntry> reportEntries, String filename,
ExcelModel excelModel, PlaceholderModel placeholderModel) {
long start = System.currentTimeMillis();
AtomicInteger rowIndex = new AtomicInteger(excelModel.getPlaceholderRow());
AtomicInteger rowIndex = new AtomicInteger(excelModel.getRedactionPlaceholderRow());
Map<Integer, Function<PlaceholderInput, String>> placeholderCellPos = excelModel.getPlaceholderCellPos();
reportEntries.forEach(entry -> {
sheet.createRow(rowIndex.get());
excelModel.getWrittenRows().add(rowIndex.get());
for (Map.Entry<Integer, Function<PlaceholderInput, String>> entry1 : placeholderCellPos.entrySet()) {
Cell cell = sheet.getRow(rowIndex.get()).createCell(entry1.getKey());
cell.setCellValue(entry1.getValue().apply(new PlaceholderInput(filename, entry, placeholderModel, null)));
cell.setCellValue(entry1.getValue()
.apply(new PlaceholderInput(filename, entry, placeholderModel, null)));
}
rowIndex.getAndIncrement();
});
log.debug("Adding rows took: {}", System.currentTimeMillis() - start);
excelModel.setPlaceholderRow(rowIndex.getAndIncrement());
return excelModel;
excelModel.setRedactionPlaceholderRow(rowIndex.getAndIncrement());
log.debug("Adding rows took: {}", System.currentTimeMillis() - start);
}
@ -218,61 +249,53 @@ public class ExcelTemplateReportGenerationService {
.replaceAll("\n", " ")
.replaceAll(" ", " ") : "";
}
// TODO in case placeholder is invalid -> do not replace with empty string but throw previus exception
return (input) -> "";
}
private void replacePlaceholdersForTextPlaceholder(Sheet sheet, String search, String replace) {
private void replacePlaceholdersForCell(Cell cell, String search, String replace) {
for (Row row : sheet) {
for (Cell cell : row) {
if (cell.getStringCellValue().contains(search)) {
String safeToUseInReplaceAllString = Pattern.quote(search);
String escapedReplace = Matcher.quoteReplacement(replace);
String str = cell.getStringCellValue().replaceAll(safeToUseInReplaceAllString, escapedReplace);
cell.setCellValue(str);
}
}
if (cell.getStringCellValue().contains(search)) {
String safeToUseInReplaceAllString = Pattern.quote(search);
String escapedReplace = Matcher.quoteReplacement(replace);
String str = cell.getStringCellValue().replaceAll(safeToUseInReplaceAllString, escapedReplace);
cell.setCellValue(str);
}
}
private void replacePlaceholdersForImagePlaceholder(XSSFWorkbook workbook, Sheet sheet,
ImagePlaceholder imagePlaceholder) throws IOException {
private void replacePlaceholdersForImagePlaceholder(SXSSFWorkbook workbook, Sheet sheet, Cell cell,
PlaceholderModel placeholderModel) throws IOException {
for (Row row : sheet) {
for (Cell cell : row) {
if (cell.getStringCellValue().contains(imagePlaceholder.getPlaceholder())) {
try (ByteArrayInputStream is = new ByteArrayInputStream(imagePlaceholder.getImage())) {
for (ImagePlaceholder imagePlaceholder : placeholderModel.getImagePlaceholders()) {
// ByteArrayInputStream scaledImage = getScaledImage(is, PixelUtil.widthUnits2Pixel((short) sheet.getColumnWidth(cell.getColumnIndex())), PixelUtil.heightUnits2Pixel(cell.getRow().getHeight()));
// is.reset();
if (cell.getStringCellValue().contains(imagePlaceholder.getPlaceholder())) {
try (ByteArrayInputStream is = new ByteArrayInputStream(imagePlaceholder.getImage())) {
double factor = calculateScale(is, PixelUtil.widthUnits2Pixel((short) sheet.getColumnWidth(cell.getColumnIndex())), PixelUtil.heightUnits2Pixel(cell.getRow()
.getHeight()));
is.reset();
double factor = calculateScale(is, PixelUtil.widthUnits2Pixel((short) sheet.getColumnWidth(cell.getColumnIndex())), PixelUtil.heightUnits2Pixel(cell.getRow()
.getHeight()));
is.reset();
int pictureIdx = workbook.addPicture(is, XSSFWorkbook.PICTURE_TYPE_JPEG);
is.reset();
int pictureIdx = workbook.addPicture(is.readAllBytes(), SXSSFWorkbook.PICTURE_TYPE_JPEG);
is.reset();
//Returns an object that handles instantiating concrete classes
CreationHelper helper = workbook.getCreationHelper();
//Create an anchor that is attached to the worksheet
ClientAnchor anchor = helper.createClientAnchor();
anchor.setAnchorType(ClientAnchor.AnchorType.MOVE_AND_RESIZE);
//Returns an object that handles instantiating concrete classes
CreationHelper helper = workbook.getCreationHelper();
//Create an anchor that is attached to the worksheet
ClientAnchor anchor = helper.createClientAnchor();
anchor.setAnchorType(ClientAnchor.AnchorType.MOVE_AND_RESIZE);
anchor.setCol1(cell.getColumnIndex());
anchor.setRow1(cell.getRowIndex());
anchor.setCol1(cell.getColumnIndex());
anchor.setRow1(cell.getRowIndex());
//Creates the top-level drawing patriarch.
Drawing drawing = sheet.createDrawingPatriarch();
//Creates the top-level drawing patriarch.
Drawing drawing = sheet.createDrawingPatriarch();
Picture picture = drawing.createPicture(anchor, pictureIdx);
picture.resize(factor);
Picture picture = drawing.createPicture(anchor, pictureIdx);
picture.resize(factor);
cell.setCellValue("");
cell.setCellValue("");
}
}
}
}
@ -301,9 +324,8 @@ public class ExcelTemplateReportGenerationService {
}
private String getPlaceholderValue(String placeholder, Dossier project, FileModel fileStatus,
Map<String, String> fileAttributePlaceholders,
Map<String, String> dossierAttributesPlaceholders) {
private String getPlaceholderValue(String placeholder, String dossierName, String filename,
PlaceholderModel placeholderModel) {
if (placeholder.equals(FORMAT_DATE_ISO_PLACEHOLDER)) {
return OffsetDateTime.now().format(FORMAT_DATE_ISO);
@ -318,23 +340,19 @@ public class ExcelTemplateReportGenerationService {
return OffsetDateTime.now().format(FORMAT_TIME_ISO);
}
if (placeholder.equals(DOSSIER_NAME_PLACEHOLDER)) {
return project.getDossierName();
return dossierName;
}
if (placeholder.equals(FILE_NAME_PLACEHOLDER)) {
return null;
return filename;
}
if (placeholderModel.getFileAttributeValueByPlaceholder().containsKey(placeholder)) {
return placeholderModel.getFileAttributeValueByPlaceholder().get(placeholder);
}
if (fileAttributePlaceholders.containsKey(placeholder)) {
String id = fileAttributePlaceholders.get(placeholder);
if (fileStatus.getFileAttributes().containsKey(id)) {
return fileStatus.getFileAttributes().get(id);
} else {
return null;
}
}
if (dossierAttributesPlaceholders.containsKey(placeholder)) {
return dossierAttributesPlaceholders.get(placeholder);
if (placeholderModel.getDossierAttributesValueByPlaceholder().containsKey(placeholder)) {
return placeholderModel.getDossierAttributesValueByPlaceholder().get(placeholder);
}
return null;
}

View File

@ -11,9 +11,12 @@ import org.springframework.stereotype.Service;
import com.iqser.red.service.redaction.report.v1.server.model.ReportRedactionEntry;
import io.micrometer.core.annotation.Timed;
@Service
public class IuclidFunctionService {
@Timed("computeIuclidFunction")
public String computeIuclidFunction(List<ReportRedactionEntry> reportRedactionEntries, String filename) {
Map<String, List<ReportRedactionEntry>> entriesPerJustification = new HashMap<>();

View File

@ -1,5 +1,18 @@
package com.iqser.red.service.redaction.report.v1.server.service;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.apache.poi.xwpf.usermodel.XWPFDocument;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.ReportTemplate;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.Dossier;
@ -22,30 +35,14 @@ import com.iqser.red.service.redaction.v1.model.RedactionLog;
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
import io.micrometer.core.annotation.Timed;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.apache.poi.xwpf.usermodel.XWPFDocument;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@Slf4j
@Service
@RequiredArgsConstructor
@SuppressWarnings("PMD")
public class ReportGenerationService {
private final ReportStorageService reportStorageService;
@ -55,12 +52,13 @@ public class ReportGenerationService {
private final DossierClient dossierClient;
private final ReportTemplateClient reportTemplateClient;
private final RedactionLogClient redactionLogClient;
private final ExcelTemplateReportGenerationService excelTemplateReportGenerationService;
private final ExcelReportGenerationService excelTemplateReportGenerationService;
private final GeneratePlaceholderService generatePlaceholderService;
@SneakyThrows
public List<StoredFileInformation> generateReport(ReportRequestMessage reportMessage) {
@Timed("generateReports")
public List<StoredFileInformation> generateReports(ReportRequestMessage reportMessage) {
List<StoredFileInformation> storedFileInformation = Collections.synchronizedList(new ArrayList<>());
@ -78,7 +76,7 @@ public class ReportGenerationService {
try (ByteArrayInputStream is = new ByteArrayInputStream(excelTemplate)) {
XSSFWorkbook readWorkbook = new XSSFWorkbook(is);
SXSSFWorkbook writeWorkbook = new SXSSFWorkbook();
for(Sheet sheet: readWorkbook){
for (Sheet sheet : readWorkbook) {
writeWorkbook.createSheet(sheet.getSheetName());
}
MultiFileWorkbook multiFileWorkbook = new MultiFileWorkbook(readWorkbook, writeWorkbook, templateId, reportTemplate.getFileName());
@ -117,29 +115,27 @@ public class ReportGenerationService {
generatePlaceholderService.resolveFileAttributeValues(fileStatus, placeholderModel);
List<ReportRedactionEntry> reportEntries = getReportEntries(reportMessage.getDossierId(), reportMessage.getFileIds()
.get(j), fileStatus.isExcluded());
var isLastFile = j == reportMessage.getFileIds().size() - 1;
for (MultiFileWorkbook multiFileWorkbook : multiFileWorkbooks) {
if(excelModel == null){
excelModel = excelTemplateReportGenerationService.caculateExcelModel(multiFileWorkbook.getReadWorkBook().getSheetAt(0));
if (excelModel == null) {
excelModel = excelTemplateReportGenerationService.calculateExcelModel(multiFileWorkbook.getReadWorkBook()
.getSheetAt(0));
}
excelModel = excelTemplateReportGenerationService.generateReport(reportEntries, placeholderModel, multiFileWorkbook.getTemplateName(), multiFileWorkbook.getWriteWorkbook(), fileStatus, excelModel);
excelTemplateReportGenerationService.generateExcelReport(reportEntries, placeholderModel, multiFileWorkbook.getTemplateName(), multiFileWorkbook.getWriteWorkbook(), dossier.getDossierName(), fileStatus, excelModel, isLastFile);
}
for (MultiFileDocument multiFileDocument : multiFileDocuments) {
wordReportGenerationService.generateReport(reportEntries, placeholderModel, multiFileDocument.getTemplateName(), multiFileDocument.getDocument(), fileStatus, dossier, isLastFile);
wordReportGenerationService.generateWordReport(reportEntries, placeholderModel, multiFileDocument.getTemplateName(), multiFileDocument.getDocument(), fileStatus, dossier, isLastFile);
}
for (ReportTemplate reportTemplate : singleFilesTemplates) {
storedFileInformation.add(createReportFromTemplate(dossier, fileStatus, placeholderModel, reportTemplate.getFileName(), reportMessage.getDownloadId(), reportEntries, reportTemplate));
}
long end = System.currentTimeMillis();
@ -174,11 +170,11 @@ public class ReportGenerationService {
try (ByteArrayInputStream is = new ByteArrayInputStream(excelTemplate)) {
XSSFWorkbook readWorkbook = new XSSFWorkbook(is);
SXSSFWorkbook writeWorkbook = new SXSSFWorkbook();
for(Sheet sheet: readWorkbook){
for (Sheet sheet : readWorkbook) {
writeWorkbook.createSheet(sheet.getSheetName());
}
var excelModel = excelTemplateReportGenerationService.caculateExcelModel(readWorkbook.getSheetAt(0));
excelTemplateReportGenerationService.generateReport(reportEntries, placeholderModel, templateName, writeWorkbook, fileStatus, excelModel);
var excelModel = excelTemplateReportGenerationService.calculateExcelModel(readWorkbook.getSheetAt(0));
excelTemplateReportGenerationService.generateExcelReport(reportEntries, placeholderModel, templateName, writeWorkbook, dossier.getDossierName(), fileStatus, excelModel, true);
byte[] template = excelTemplateReportGenerationService.toByteArray(writeWorkbook);
String storageId = reportStorageService.storeObject(downloadId, template);
return new StoredFileInformation(fileStatus.getId(), storageId, ReportType.EXCEL_TEMPLATE_SINGLE_FILE, reportTemplate.getTemplateId());
@ -189,7 +185,7 @@ public class ReportGenerationService {
byte[] wordTemplate = reportStorageService.getReportTemplate(reportTemplate.getStorageId());
try (ByteArrayInputStream is = new ByteArrayInputStream(wordTemplate)) {
XWPFDocument doc = new XWPFDocument(is);
wordReportGenerationService.generateReport(reportEntries, placeholderModel, templateName, doc, fileStatus, dossier, true);
wordReportGenerationService.generateWordReport(reportEntries, placeholderModel, templateName, doc, fileStatus, dossier, true);
byte[] template = wordReportGenerationService.toByteArray(doc);
String storageId = reportStorageService.storeObject(downloadId, template);
return new StoredFileInformation(fileStatus.getId(), storageId, ReportType.WORD_SINGLE_FILE, reportTemplate.getTemplateId());
@ -200,6 +196,7 @@ public class ReportGenerationService {
}
@Timed("getReportEntries")
private List<ReportRedactionEntry> getReportEntries(String dossierId, String fileId, boolean isExcluded) {
if (isExcluded) {
@ -228,7 +225,4 @@ public class ReportGenerationService {
return redactionLogConverterService.convertAndSort(redactionLog, legalBasisMappings);
}
}

View File

@ -36,7 +36,7 @@ public class ReportMessageReceiver {
long start = System.currentTimeMillis();
log.info("Start generating reports for downloadId {}", reportMessage.getDownloadId());
List<StoredFileInformation> storedFileInformation = reportGenerationService.generateReport(reportMessage);
List<StoredFileInformation> storedFileInformation = reportGenerationService.generateReports(reportMessage);
addToReportResultQueue(reportMessage.getUserId(), reportMessage.getDownloadId(), storedFileInformation, 1);
long end = System.currentTimeMillis();

View File

@ -34,16 +34,15 @@ import static com.iqser.red.service.redaction.report.v1.server.service.Placehold
@Slf4j
@Service
@RequiredArgsConstructor
@SuppressWarnings("PMD")
public class WordReportGenerationService {
private final IuclidFunctionService iuclidFunctionService;
@Timed("generateWordReport")
public XWPFDocument generateReport(List<ReportRedactionEntry> reportEntries, PlaceholderModel placeholderModel,
String reportTemplateName, XWPFDocument doc, FileModel fileModel,
Dossier dossier, boolean isLastFile) {
public XWPFDocument generateWordReport(List<ReportRedactionEntry> reportEntries, PlaceholderModel placeholderModel,
String reportTemplateName, XWPFDocument doc, FileModel fileModel,
Dossier dossier, boolean isLastFile) {
long start = System.currentTimeMillis();
@ -53,26 +52,19 @@ public class WordReportGenerationService {
replaceImagePlaceholders(doc, imagePlaceholder);
}
long t1 = System.currentTimeMillis();
XWPFTable table = getRedactionTable(doc);
var placeholderFunctions = computePlaceholderPos(table, placeholderModel);
addTableRows(table, reportEntries, fileModel.getFilename(), placeholderModel, placeholderFunctions);
long t2 = System.currentTimeMillis();
log.debug("Table time: {}", (t2 - t1));
t1 = System.currentTimeMillis();
var placeholderFunctions = computePlaceholderPos(table, placeholderModel);
addRedactionEntryRows(table, reportEntries, fileModel.getFilename(), placeholderModel, placeholderFunctions);
replaceTextPlaceholders(doc, placeholderModel, dossier.getDossierName(), fileModel.getFilename(), table, reportEntries);
if (isLastFile) {
removePlaceholdersRow(table);
}
t2 = System.currentTimeMillis();
log.debug("Text time: {}", (t2 - t1));
long end = System.currentTimeMillis();
log.info("Report Generation took: {} for file with id {}, pageCount: {}, redactionLogEntryCount: {}, reportName: {}, className: {}", end - start, fileModel.getId(), fileModel.getNumberOfPages(), reportEntries.size(), reportTemplateName, getClass().getSimpleName());
log.info("Report Generation took: {} for file with id {}, pageCount: {}, redactionLogEntryCount: {}, reportName: {}, className: {}", System.currentTimeMillis() - start, fileModel.getId(), fileModel.getNumberOfPages(), reportEntries.size(), reportTemplateName, getClass().getSimpleName());
return doc;
} catch (Exception e) {
@ -175,6 +167,7 @@ public class WordReportGenerationService {
}
@Timed("word-replaceTextPlaceholders")
public void replaceTextPlaceholders(XWPFDocument doc, PlaceholderModel placeholderModel, String dossierName,
String fileName, XWPFTable tableToSkip,
List<ReportRedactionEntry> reportRedactionEntries) {
@ -267,6 +260,7 @@ public class WordReportGenerationService {
}
@Timed("word-computePlaceholderPos")
private PlaceHolderFunctions computePlaceholderPos(XWPFTable table, PlaceholderModel placeholderModel) {
Set<String> foundPlaceHolder = new HashSet<>();
@ -288,23 +282,25 @@ public class WordReportGenerationService {
}
private void addTableRows(XWPFTable table, List<ReportRedactionEntry> reportEntries, String filename,
PlaceholderModel placeholderModel, PlaceHolderFunctions placeHolderFunctions) {
@Timed("word-addRedactionEntryRows")
private void addRedactionEntryRows(XWPFTable table, List<ReportRedactionEntry> reportEntries, String filename,
PlaceholderModel placeholderModel, PlaceHolderFunctions placeHolderFunctions) {
if (table == null) {
return;
}
if (placeHolderFunctions.getFoundPlaceHolder().contains(SEEDS_FUNCTION_REDACTION_GROUPED_BY_JUSTIFICATION_PAGES_PLACEHOLDER)) {
if (placeHolderFunctions.getFoundPlaceHolder()
.contains(SEEDS_FUNCTION_REDACTION_GROUPED_BY_JUSTIFICATION_PAGES_PLACEHOLDER)) {
var redactionsPerJustification = getRedactionsPerJustification(reportEntries);
for (Map.Entry<String, List<ReportRedactionEntry>> entry : redactionsPerJustification.entrySet()) {
XWPFTableRow row = table.createRow();
for (Map.Entry<Integer, Function<PlaceholderInput, String>> entry1 : placeHolderFunctions.getFunctionPerPlaceHolder()
.entrySet()) {
setText(row.getCell(entry1.getKey()), entry1.getValue().apply(new PlaceholderInput(filename, null, placeholderModel, entry)));
setText(row.getCell(entry1.getKey()), entry1.getValue()
.apply(new PlaceholderInput(filename, null, placeholderModel, entry)));
}
}
} else {
@ -312,7 +308,8 @@ public class WordReportGenerationService {
XWPFTableRow row = table.createRow();
for (Map.Entry<Integer, Function<PlaceholderInput, String>> entry1 : placeHolderFunctions.getFunctionPerPlaceHolder()
.entrySet()) {
setText(row.getCell(entry1.getKey()), entry1.getValue().apply(new PlaceholderInput(filename, entry, placeholderModel, null)));
setText(row.getCell(entry1.getKey()), entry1.getValue()
.apply(new PlaceholderInput(filename, entry, placeholderModel, null)));
}
});
}

View File

@ -3,13 +3,7 @@ package com.iqser.red.service.redaction.report.v1.server;
import com.amazonaws.services.s3.AmazonS3;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DossierAttributeConfig;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.ReportTemplate;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.Dossier;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.DossierAttribute;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.DossierAttributeType;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileAttributeConfig;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileAttributeType;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.legalbasis.LegalBasis;
import com.iqser.red.service.redaction.report.v1.server.client.DossierAttributesClient;
@ -20,7 +14,7 @@ import com.iqser.red.service.redaction.report.v1.server.configuration.MessagingC
import com.iqser.red.service.redaction.report.v1.server.model.ImagePlaceholder;
import com.iqser.red.service.redaction.report.v1.server.model.PlaceholderModel;
import com.iqser.red.service.redaction.report.v1.server.model.ReportRedactionEntry;
import com.iqser.red.service.redaction.report.v1.server.service.ExcelTemplateReportGenerationService;
import com.iqser.red.service.redaction.report.v1.server.service.ExcelReportGenerationService;
import com.iqser.red.service.redaction.report.v1.server.service.GeneratePlaceholderService;
import com.iqser.red.service.redaction.report.v1.server.service.RedactionLogConverterService;
import com.iqser.red.service.redaction.report.v1.server.service.WordReportGenerationService;
@ -32,7 +26,6 @@ import org.apache.commons.io.IOUtils;
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.apache.poi.xwpf.usermodel.XWPFDocument;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
@ -42,8 +35,6 @@ import org.springframework.core.io.ClassPathResource;
import org.springframework.test.context.junit4.SpringRunner;
import java.io.FileOutputStream;
import java.io.IOException;
import java.time.OffsetDateTime;
import java.util.*;
import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.DOSSIER_NAME_PLACEHOLDER;
@ -56,7 +47,6 @@ import static com.iqser.red.service.redaction.report.v1.server.service.Placehold
import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.SEEDS_FUNCTION_JUSTIFICATION_PLACEHOLDER;
import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.SEEDS_FUNCTION_REDACTION_GROUPED_BY_JUSTIFICATION_PAGES_PLACEHOLDER;
import static com.iqser.red.service.redaction.report.v1.server.utils.OsUtils.getTemporaryDirectory;
import static org.mockito.Mockito.when;
import static org.springframework.boot.test.context.SpringBootTest.WebEnvironment.RANDOM_PORT;
@RunWith(SpringRunner.class)
@ -97,7 +87,7 @@ public class RedactionReportIntegrationTest {
private DossierAttributesClient dossierAttributesClient;
@Autowired
private ExcelTemplateReportGenerationService excelTemplateReportGenerationService;
private ExcelReportGenerationService excelTemplateReportGenerationService;
@Autowired
private GeneratePlaceholderService generatePlaceholderService;
@ -127,7 +117,7 @@ public class RedactionReportIntegrationTest {
List.of(new ImagePlaceholder("{{dossier.attribute.Signature}}",IOUtils.toByteArray(imageResource.getInputStream()))));
XWPFDocument doc = new XWPFDocument(wordTemplateResource.getInputStream());
wordReportGenerationService.generateReport(reportEntries, placeholders, "test", doc, fileModel, dossier, true);
wordReportGenerationService.generateWordReport(reportEntries, placeholders, "test", doc, fileModel, dossier, true);
byte[] wordReport = wordReportGenerationService.toByteArray(doc);
try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/Justification Appendix A1_justification.docx")) {
fileOutputStream.write(wordReport);
@ -159,7 +149,7 @@ public class RedactionReportIntegrationTest {
List.of(new ImagePlaceholder("{{dossier.attribute.Signature}}",IOUtils.toByteArray(imageResource.getInputStream()))));
XWPFDocument doc = new XWPFDocument(wordTemplateResource.getInputStream());
wordReportGenerationService.generateReport(reportEntries, placeholders, "test", doc, fileModel, dossier, true);
wordReportGenerationService.generateWordReport(reportEntries, placeholders, "test", doc, fileModel, dossier, true);
byte[] wordReport = wordReportGenerationService.toByteArray(doc);
try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/Justification Appendix A2_justification.docx")) {
fileOutputStream.write(wordReport);
@ -183,7 +173,7 @@ public class RedactionReportIntegrationTest {
var placeholders = buildPlaceHolderModel(new HashMap<>(), Map.of("{{file.attribute.Path}}", "Path"), List.of());
wordReportGenerationService.generateReport(reportEntries, placeholders, "test", doc, fileStatus, dossier, true);
wordReportGenerationService.generateWordReport(reportEntries, placeholders, "test", doc, fileStatus, dossier, true);
byte[] report = wordReportGenerationService.toByteArray(doc);
try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/IUCLID_Template_justification.docx")) {
@ -208,7 +198,7 @@ public class RedactionReportIntegrationTest {
var placeholders = buildPlaceHolderModel(new HashMap<>(), Map.of("{{file.attribute.Path}}", "Path"), List.of());
wordReportGenerationService.generateReport(reportEntries, placeholders, "test", doc, fileStatus, dossier, true);
wordReportGenerationService.generateWordReport(reportEntries, placeholders, "test", doc, fileStatus, dossier, true);
byte[] report = wordReportGenerationService.toByteArray(doc);
try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/seeds-report.docx")) {
@ -238,8 +228,8 @@ public class RedactionReportIntegrationTest {
var placeholders = buildPlaceHolderModel(new HashMap<>(), Map.of("{{file.attribute.Path}}", "Path"), List.of());
wordReportGenerationService.generateReport(reportEntries, placeholders, "test", doc, fileStatus, dossier, false);
wordReportGenerationService.generateReport(reportEntriesSecondFile, placeholders, "test", doc, fileModelSecondFile, dossier, true);
wordReportGenerationService.generateWordReport(reportEntries, placeholders, "test", doc, fileStatus, dossier, false);
wordReportGenerationService.generateWordReport(reportEntriesSecondFile, placeholders, "test", doc, fileModelSecondFile, dossier, true);
byte[] report = wordReportGenerationService.toByteArray(doc);
try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/seeds-report-multifile.docx")) {
@ -274,7 +264,7 @@ public class RedactionReportIntegrationTest {
List.of(new ImagePlaceholder("{{dossier.attribute.Signature}}",IOUtils.toByteArray(imageResource.getInputStream()))));
XWPFDocument doc = new XWPFDocument(wordTemplateResource.getInputStream());
wordReportGenerationService.generateReport(reportEntries, placeholders, "test", doc, fileModel, dossier, true);
wordReportGenerationService.generateWordReport(reportEntries, placeholders, "test", doc, fileModel, dossier, true);
byte[] wordReport = wordReportGenerationService.toByteArray(doc);
try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/6464 appendix_b EFSA dRAR_justification.docx")) {
fileOutputStream.write(wordReport);
@ -297,10 +287,10 @@ public class RedactionReportIntegrationTest {
var placeholders = buildPlaceHolderModel(new HashMap<>(), new HashMap<>(), List.of());
XSSFWorkbook readWorkbook = new XSSFWorkbook(templateResource.getInputStream());
var excelModel = excelTemplateReportGenerationService.caculateExcelModel(readWorkbook.getSheetAt(0));
var excelModel = excelTemplateReportGenerationService.calculateExcelModel(readWorkbook.getSheetAt(0));
SXSSFWorkbook writeWorkbook = new SXSSFWorkbook();
writeWorkbook.createSheet("Sheet1");
excelTemplateReportGenerationService.generateReport(reportEntries, placeholders, "test", writeWorkbook, fileModel, excelModel);
excelTemplateReportGenerationService.generateExcelReport(reportEntries, placeholders, "test", writeWorkbook, "dossierName" ,fileModel, excelModel, true);
byte[] excelTemplateReport3 = excelTemplateReportGenerationService.toByteArray(writeWorkbook);
@ -326,15 +316,15 @@ public class RedactionReportIntegrationTest {
var placeholders = buildPlaceHolderModel(new HashMap<>(), new HashMap<>(), List.of());
XSSFWorkbook readWorkbook = new XSSFWorkbook(templateResource.getInputStream());
var excelModel = excelTemplateReportGenerationService.caculateExcelModel(readWorkbook.getSheetAt(0));
var excelModel = excelTemplateReportGenerationService.calculateExcelModel(readWorkbook.getSheetAt(0));
SXSSFWorkbook writeWorkbook = new SXSSFWorkbook();
writeWorkbook.createSheet("Sheet1");
excelModel = excelTemplateReportGenerationService.generateReport(reportEntries, placeholders, "test", writeWorkbook, fileModel, excelModel);
excelTemplateReportGenerationService.generateExcelReport(reportEntries, placeholders, "test", writeWorkbook, "dossierName" ,fileModel, excelModel, false);
RedactionLog redactionLogSecondFile = objectMapper.readValue(new ClassPathResource("files/excelReportRedactionLog.json").getInputStream(), RedactionLog.class);
List<ReportRedactionEntry> reportEntriesSecondFile = redactionLogConverterService.convertAndSort(redactionLogSecondFile, legalBasisMapping);
FileModel fileModelSecondFile = FileModel.builder().filename("secondFile").build();
excelTemplateReportGenerationService.generateReport(reportEntriesSecondFile, placeholders, "test", writeWorkbook, fileModelSecondFile, excelModel);
excelTemplateReportGenerationService.generateExcelReport(reportEntriesSecondFile, placeholders, "test", writeWorkbook,"dossierName" , fileModelSecondFile, excelModel, true);
byte[] excelTemplateReport3 = excelTemplateReportGenerationService.toByteArray(writeWorkbook);
@ -344,6 +334,37 @@ public class RedactionReportIntegrationTest {
}
@Test
@SneakyThrows
public void testExcelPlaceholders() {
RedactionLog redactionLog = objectMapper.readValue(new ClassPathResource("files/redactionLog.json").getInputStream(), RedactionLog.class);
List<LegalBasis> legalBasisMapping = objectMapper.readValue(new ClassPathResource("files/legalBasisMapping.json").getInputStream(), new TypeReference<>() {});
List<ReportRedactionEntry> reportEntries = redactionLogConverterService.convertAndSort(redactionLog, legalBasisMapping);
var imageResource = new ClassPathResource("files/exampleImage.jpg");
FileModel fileModel = FileModel.builder().filename("filename").build();
ClassPathResource templateResource = new ClassPathResource("templates/Excel Report_PlaceholderTest.xlsx");
var placeholders = buildPlaceHolderModel(Map.of("{{dossier_attribute.test1}}", "replaced_dossier_test1"), Map.of("{{file_attribute.test1}}", "replaced_file_test1", "{{file_attribute.test2}}", "replaced_file_test2"), List.of(new ImagePlaceholder("{{dossier.attribute.Signature}}",IOUtils.toByteArray(imageResource.getInputStream()))));
XSSFWorkbook readWorkbook = new XSSFWorkbook(templateResource.getInputStream());
var excelModel = excelTemplateReportGenerationService.calculateExcelModel(readWorkbook.getSheetAt(0));
SXSSFWorkbook writeWorkbook = new SXSSFWorkbook();
writeWorkbook.createSheet("Sheet1");
excelTemplateReportGenerationService.generateExcelReport(reportEntries, placeholders, "test", writeWorkbook,"dossierName" , fileModel, excelModel, true);
byte[] excelTemplateReport3 = excelTemplateReportGenerationService.toByteArray(writeWorkbook);
try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/PlaceholderTest_justification.xlsx")) {
fileOutputStream.write(excelTemplateReport3);
}
}
private PlaceholderModel buildPlaceHolderModel(Map<String, String> dossierAttributes, Map<String, String> fileAttributes, List<ImagePlaceholder> imagePlaceholders){
var defaultPlaceHolder = new HashSet(Set.of(FILE_NAME_PLACEHOLDER, FORMAT_DATE_ISO_PLACEHOLDER, FORMAT_DATE_GER_PLACEHOLDER, FORMAT_DATE_ENG_PLACEHOLDER, FORMAT_TIME_ISO_PLACEHOLDER, DOSSIER_NAME_PLACEHOLDER, IUCLID_FUNCTION_PLACEHOLDER, SEEDS_FUNCTION_REDACTION_GROUPED_BY_JUSTIFICATION_PAGES_PLACEHOLDER, SEEDS_FUNCTION_JUSTIFICATION_PLACEHOLDER));