Marmelator/redaction-report-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 129 Wiki Activity

Pull request #147: RED-4036 - improved report service performance, added some logs

Browse Source 
Merge in RED/redaction-report-service from RED-4036 to master

* commit 'f34f113d95a22f1ac6e6b5206a4dbef36d45ea42':
  RED-4036: Cleanup logs
  RED-4036: Fixed singleFile excel reports
  RED-4036: Copy colmn style and non entry cells in excel reports
  RED-4036: Added readWorkbook and writeWorkbook as you cannot read from  SXSSFWorkbook
  RED-4036: Changed excel to use SXSSFWorkbook
  RED-4036: Added more logs
  RED-4036: Fixed bug in excel optimization
  RED-4036: Optimized excel
  RED-4036: Ignore non entry placeholders in excel
  RED-4036: Revert parallel processing
  RED-4036: Wait for executor timeout
  RED-4036 - improved report service performance, added some logs
  RED-4036 - improved report service performance, added some logs
This commit is contained in:
Timo Bejan 2022-05-16 15:17:48 +02:00 committed by Dominique Eiflaender
commit bc6f8d5473
19 changed files with 739 additions and 425 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
redaction-report-service-image-v1/pom.xml
View File

@ -5,7 +5,7 @@
<parent>
<groupId>com.iqser.red</groupId>
<artifactId>platform-docker-dependency</artifactId>
<version>1.1.0</version>
<version>1.2.0</version>
<relativePath />
</parent>
<modelVersion>4.0.0</modelVersion>
@ -41,7 +41,7 @@
<artifactId>docker-maven-plugin</artifactId>
</plugin>
</plugins>
<pluginManagement>
<plugins>
<plugin>

2
redaction-report-service-v1/pom.xml
View File

@ -27,7 +27,7 @@
<dependency>
<groupId>com.iqser.red</groupId>
<artifactId>platform-commons-dependency</artifactId>
<version>1.11.0</version>
<version>1.13.0</version>
<scope>import</scope>
<type>pom</type>
</dependency>

4
redaction-report-service-v1/redaction-report-service-server-v1/pom.xml
View File

@ -14,8 +14,8 @@
<version>1.0-SNAPSHOT</version>
<properties>
<persistence-service.version>1.160.0</persistence-service.version>
<redaction-service.version>3.108.0</redaction-service.version>
<persistence-service.version>1.169.0</persistence-service.version>
<redaction-service.version>3.109.0</redaction-service.version>
</properties>
<dependencies>

15
redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/model/CellIdentifier.java Normal file
View File

@ -0,0 +1,15 @@
package com.iqser.red.service.redaction.report.v1.server.model;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@NoArgsConstructor
@AllArgsConstructor
public class CellIdentifier {
private int rowIndex;
private int columnIndex;
}

24
redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/model/ExcelModel.java Normal file
View File

@ -0,0 +1,24 @@
package com.iqser.red.service.redaction.report.v1.server.model;
import java.util.HashMap;
import java.util.Map;
import java.util.function.Function;
import org.apache.poi.ss.usermodel.Cell;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@NoArgsConstructor
@AllArgsConstructor
public class ExcelModel {
private Map<Integer, Function<TextPlaceholderInput, String>> placeholderCellPos;
private int placeholderRow;
private Map<Integer, Integer> cellWidths = new HashMap<>();
private Map<CellIdentifier, Cell> cellsToCopy = new HashMap<>();
private boolean copyDone;
}

8
redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/model/MultiFileDocument.java
View File

@ -1,13 +1,8 @@
package com.iqser.red.service.redaction.report.v1.server.model;
import java.util.List;
import org.apache.poi.xwpf.usermodel.XWPFDocument;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel;
import lombok.AllArgsConstructor;
import lombok.Data;
import org.apache.poi.xwpf.usermodel.XWPFDocument;
@Data
@AllArgsConstructor
@ -15,5 +10,6 @@ public class MultiFileDocument {
private XWPFDocument document;
private String templateId;
private String templateName;
}

6
redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/model/MultiFileWorkbook.java
View File

@ -2,13 +2,17 @@ package com.iqser.red.service.redaction.report.v1.server.model;
import lombok.AllArgsConstructor;
import lombok.Data;
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
@Data
@AllArgsConstructor
public class MultiFileWorkbook {
private XSSFWorkbook workbook;
private XSSFWorkbook readWorkBook;
private SXSSFWorkbook writeWorkbook;
private String templateId;
private String templateName;
}

9
redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/model/PlaceholderInput.java Normal file
View File

@ -0,0 +1,9 @@
package com.iqser.red.service.redaction.report.v1.server.model;
import java.util.List;
import java.util.Map;
public interface PlaceholderInput {
}

20
redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/model/PlaceholderModel.java Normal file
View File

@ -0,0 +1,20 @@
package com.iqser.red.service.redaction.report.v1.server.model;
import lombok.AllArgsConstructor;
import lombok.Data;
import java.util.List;
import java.util.Map;
@Data
@AllArgsConstructor
public class PlaceholderModel {
private List<String> placeholders;
private List<ImagePlaceholder> imagePlaceholders;
private Map<String, String> dossierAttributesPlaceholder;
private Map<String, String> fileAttributePlaceHolders;
}

13
redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/model/TextPlaceholderInput.java Normal file
View File

@ -0,0 +1,13 @@
package com.iqser.red.service.redaction.report.v1.server.model;
import lombok.AllArgsConstructor;
import lombok.Data;
@Data
@AllArgsConstructor
public class TextPlaceholderInput implements PlaceholderInput {
private String filename;
private ReportRedactionEntry entry;
}

241
redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ExcelTemplateReportGenerationService.java
View File

@ -29,21 +29,30 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Base64;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Function;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.imageio.ImageIO;
import com.iqser.red.service.redaction.report.v1.server.model.CellIdentifier;
import com.iqser.red.service.redaction.report.v1.server.model.ExcelModel;
import com.iqser.red.service.redaction.report.v1.server.model.PlaceholderModel;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.CellStyle;
import org.apache.poi.ss.usermodel.ClientAnchor;
import org.apache.poi.ss.usermodel.CreationHelper;
import org.apache.poi.ss.usermodel.Drawing;
import org.apache.poi.ss.usermodel.Picture;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.springframework.stereotype.Service;
@ -57,6 +66,7 @@ import com.iqser.red.service.redaction.report.v1.server.client.DossierAttributes
import com.iqser.red.service.redaction.report.v1.server.client.FileAttributesConfigClient;
import com.iqser.red.service.redaction.report.v1.server.model.ImagePlaceholder;
import com.iqser.red.service.redaction.report.v1.server.model.ReportRedactionEntry;
import com.iqser.red.service.redaction.report.v1.server.model.TextPlaceholderInput;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
@ -65,156 +75,115 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@RequiredArgsConstructor
@SuppressWarnings("PMD")
public class ExcelTemplateReportGenerationService {
private final DossierAttributesClient dossierAttributesClient;
private final DossierAttributesConfigClient dossierAttributesConfigClient;
private final FileAttributesConfigClient fileAttributesClient;
@SuppressWarnings("checkstyle:ParameterAssignment")
public ExcelModel generateReport(List<ReportRedactionEntry> reportEntries, PlaceholderModel placeholderModel,
String reportTemplateName, SXSSFWorkbook workbook, FileModel fileStatus,
Dossier dossier, boolean isLastFile, ExcelModel excelModel) {
public void generateReport(List<ReportRedactionEntry> reportEntries, String dossierTemplateId,
XSSFWorkbook workbook, FileModel fileStatus, Dossier dossier, boolean isLastFile) {
List<String> placeholders = getDefaultPlaceholders();
List<ImagePlaceholder> imagePlaceholders = new ArrayList<>();
var dossierAttributes = dossierAttributesClient.getDossierAttributes(dossier.getId());
var dossierAttributesConfig = dossierAttributesConfigClient.getDossierAttributes(dossierTemplateId);
Map<String, String> dossierAttributesPlaceholder = new HashMap<>(); //KEY:placeholder, VALUE:value
for (DossierAttributeConfig attributeConfig : dossierAttributesConfig) {
for (DossierAttribute dossierAttribute : dossierAttributes) {
if (dossierAttribute.getDossierAttributeConfigId().equals(attributeConfig.getId())) {
if (attributeConfig.getType().equals(DossierAttributeType.IMAGE)) {
if (dossierAttribute.getValue() != null) {
if (dossierAttribute.getValue().startsWith("data:")) {
imagePlaceholders.add(new ImagePlaceholder(attributeConfig.getPlaceholder(), Base64.getDecoder()
.decode(dossierAttribute.getValue().split(",")[1])));
} else {
imagePlaceholders.add(new ImagePlaceholder(attributeConfig.getPlaceholder(), Base64.getDecoder()
.decode(dossierAttribute.getValue())));
}
}
} else {
if (dossierAttribute.getValue() == null) {
dossierAttributesPlaceholder.put(attributeConfig.getPlaceholder(), "");
} else {
dossierAttributesPlaceholder.put(attributeConfig.getPlaceholder(), dossierAttribute.getValue());
}
}
}
}
if (!dossierAttributesPlaceholder.containsKey(attributeConfig.getPlaceholder())) {
dossierAttributesPlaceholder.put(attributeConfig.getPlaceholder(), "");
}
}
Map<String, String> fileAttributePlaceholders = getFileAttributePlaceholders(dossierTemplateId);
placeholders.addAll(fileAttributePlaceholders.keySet());
placeholders.addAll(dossierAttributesPlaceholder.keySet());
long start = System.currentTimeMillis();
try {
for (Sheet sheet : workbook) {
addEntryRows(sheet, reportEntries, fileStatus.getFilename(), isLastFile);
for (ImagePlaceholder imagePlaceholder : imagePlaceholders) {
replacePlaceholdersForImagePlaceholder(workbook, sheet, imagePlaceholder);
}
for (String placeholder : placeholders) {
String placeholderValue = getPlaceholderValue(placeholder, dossier, fileStatus, fileAttributePlaceholders, dossierAttributesPlaceholder);
if (placeholderValue != null) {
replacePlaceholdersForTextPlaceholder(sheet, placeholder, placeholderValue);
if(!excelModel.isCopyDone()) {
for (Map.Entry<Integer, Integer> colummWidthEntry : excelModel.getCellWidths().entrySet()) {
sheet.setColumnWidth(colummWidthEntry.getKey(), colummWidthEntry.getValue());
}
Set<Integer> createdRows = new HashSet<>();
for (Map.Entry<CellIdentifier, Cell> cellsToCopyEntry : excelModel.getCellsToCopy().entrySet()) {
if (!createdRows.contains(cellsToCopyEntry.getKey().getRowIndex())) {
sheet.createRow(cellsToCopyEntry.getKey().getRowIndex());
createdRows.add(cellsToCopyEntry.getKey().getRowIndex());
}
var createdCell = sheet.getRow(cellsToCopyEntry.getKey().getRowIndex())
.createCell(cellsToCopyEntry.getKey().getColumnIndex());
createdCell.setCellValue(cellsToCopyEntry.getValue().getStringCellValue());
CellStyle newCellStyle = workbook.createCellStyle();
newCellStyle.cloneStyleFrom(cellsToCopyEntry.getValue().getCellStyle());
createdCell.setCellStyle(newCellStyle);
}
excelModel.setCopyDone(true);
}
excelModel = addEntryRows(sheet, reportEntries, fileStatus.getFilename(), isLastFile, excelModel);
// for (ImagePlaceholder imagePlaceholder : placeholderModel.getImagePlaceholders()){
// replacePlaceholdersForImagePlaceholder(workbook, sheet, imagePlaceholder);
// }
// for (String placeholder : placeholderModel.getPlaceholders()) {
// String placeholderValue = getPlaceholderValue(placeholder, dossier, fileStatus, placeholderModel.getFileAttributePlaceHolders(), placeholderModel.getDossierAttributesPlaceholder());
// if (placeholderValue != null) {
// replacePlaceholdersForTextPlaceholder(sheet, placeholder, placeholderValue);
// }
// }
}
} catch (IOException e) {
throw new RuntimeException(e);
long end = System.currentTimeMillis();
log.info("Report Generation took: {} for file with id {}, pageCount: {}, redactionLogEntryCount: {}, reportName: {}, className: {}", end - start, fileStatus.getId(), fileStatus.getNumberOfPages(), reportEntries.size(), reportTemplateName, getClass().getSimpleName());
} catch (Exception e) {
throw new RuntimeException(e);
}
return excelModel;
}
private List<String> getDefaultPlaceholders() {
List<String> defPlaceholders = new ArrayList<>();
defPlaceholders.addAll(Arrays.asList(FILE_NAME_PLACEHOLDER, FORMAT_DATE_ISO_PLACEHOLDER, FORMAT_DATE_GER_PLACEHOLDER, FORMAT_DATE_ENG_PLACEHOLDER, FORMAT_TIME_ISO_PLACEHOLDER, DOSSIER_NAME_PLACEHOLDER));
return defPlaceholders;
}
private Map<String, String> getFileAttributePlaceholders(String dossierTemplateId) {
Map<String, String> fileAttributePlaceholders = new HashMap<>();
var fileAttributesConfig = fileAttributesClient.getFileAttributeConfigs(dossierTemplateId);
fileAttributesConfig.forEach(fileAttributeConfig -> {
fileAttributePlaceholders.put(fileAttributeConfig.getPlaceholder(), fileAttributeConfig.getId());
});
return fileAttributePlaceholders;
}
private void addEntryRows(Sheet sheet, List<ReportRedactionEntry> reportEntries, String filename,
boolean isLastFile) {
if (sheet == null) {
return;
}
if (sheet.getLastRowNum() == -1 && sheet.getRow(0) == null) {
// This is the case when the sheet is empty!
return;
}
Map<Integer, String> placeholderCellPos = new HashMap<>();
public ExcelModel caculateExcelModel(Sheet sheet) {
long start = System.currentTimeMillis();
Map<Integer, Function<TextPlaceholderInput, String>> placeholderCellPos = new HashMap<>();
Map<Integer, Integer> columnWidths = new HashMap<>();
Map<CellIdentifier, Cell> cellsToCopy = new HashMap<>();
int placeholderRow = -1;
for (int j = 0; j < sheet.getLastRowNum() + 1; j++) {
Row actualRow = sheet.getRow(j);
if (actualRow != null) {
for (int i = 0; i < actualRow.getLastCellNum(); i++) {
Cell cell = sheet.getRow(j).getCell(i);
if (cell != null && containsRedactionPlaceholder(cell.getStringCellValue())) {
placeholderCellPos.put(i, cell.getStringCellValue());
placeholderRow = j;
if (cell != null) {
if (containsRedactionPlaceholder(cell.getStringCellValue())) {
int columnWidth = sheet.getColumnWidth(i);
columnWidths.put(i, columnWidth);
placeholderCellPos.put(i, getFunctionForPlaceHolder(cell.getStringCellValue()));
placeholderRow = j;
} else {
cellsToCopy.put(new CellIdentifier(j, i), cell);
}
}
}
}
}
log.debug("Calculate Placeholder Cells took: {}", System.currentTimeMillis() - start);
return new ExcelModel(placeholderCellPos, placeholderRow, columnWidths, cellsToCopy, false);
}
AtomicInteger rowIndex = new AtomicInteger(placeholderRow);
if (rowIndex.get() != -1) {
if (isLastFile) {
if (reportEntries.size() > 1) {
sheet.shiftRows(rowIndex.get(), rowIndex.get() + reportEntries.size() - 1, reportEntries.size() - 1, true, true);
}
} else {
if(!reportEntries.isEmpty()) {
sheet.shiftRows(rowIndex.get(), rowIndex.get() + reportEntries.size(), reportEntries.size(), true, true);
}
@SuppressWarnings("checkstyle:ParameterAssignment")
private ExcelModel addEntryRows(Sheet sheet, List<ReportRedactionEntry> reportEntries, String filename,
boolean isLastFile, ExcelModel excelModel) {
long start = System.currentTimeMillis();
AtomicInteger rowIndex = new AtomicInteger(excelModel.getPlaceholderRow());
Map<Integer, Function<TextPlaceholderInput, String>> placeholderCellPos = excelModel.getPlaceholderCellPos();
reportEntries.forEach(entry -> {
sheet.createRow(rowIndex.get());
for (Map.Entry<Integer, Function<TextPlaceholderInput, String>> entry1 : placeholderCellPos.entrySet()) {
Cell cell = sheet.getRow(rowIndex.get()).createCell(entry1.getKey());
cell.setCellValue(entry1.getValue().apply(new TextPlaceholderInput(filename, entry)));
}
rowIndex.getAndIncrement();
});
log.debug("Adding rows took: {}", System.currentTimeMillis() - start);
if(reportEntries.isEmpty() && isLastFile) {
sheet.createRow(rowIndex.get());
for (Map.Entry<Integer, String> entry1 : placeholderCellPos.entrySet()) {
sheet.getRow(rowIndex.get()).createCell(entry1.getKey()).setCellValue("");
}
} else {
reportEntries.forEach(entry -> {
sheet.createRow(rowIndex.get());
for (Map.Entry<Integer, String> entry1 : placeholderCellPos.entrySet()) {
sheet.getRow(rowIndex.get()).createCell(entry1.getKey()).setCellValue(replaceTextPlaceholderWithEntries(entry, filename, entry1.getValue()));
}
rowIndex.getAndIncrement();
});
}
}
excelModel.setPlaceholderRow(rowIndex.getAndIncrement());
return excelModel;
}
@ -224,39 +193,43 @@ public class ExcelTemplateReportGenerationService {
}
private String replaceTextPlaceholderWithEntries(ReportRedactionEntry entry, String filename, String placeholder) {
private Function<TextPlaceholderInput, String> getFunctionForPlaceHolder(String placeholder) {
if (placeholder.equals(FILE_NAME_PLACEHOLDER)) {
return filename;
return TextPlaceholderInput::getFilename;
}
if (placeholder.equals(PAGE_PLACEHOLDER)) {
return String.valueOf(entry.getPage());
return (input) -> String.valueOf(input.getEntry().getPage());
}
if (placeholder.equals(PARAGRAPH_PLACEHOLDER)) {
return entry.getSection();
return (input) -> input.getEntry().getSection();
}
if (placeholder.equals(JUSTIFICATION_PLACEHOLDER)) {
return entry.getJustification();
return (input) -> input.getEntry().getJustification();
}
if (placeholder.equals(JUSTIFICATION_PARAGRAPH_PLACEHOLDER)) {
return entry.getJustificationParagraph();
return (input) -> input.getEntry().getJustificationParagraph();
}
if (placeholder.equals(JUSTIFICATION_REASON_PLACEHOLDER)) {
return entry.getJustificationReason();
return (input) -> input.getEntry().getJustificationReason();
}
if (placeholder.equals(JUSTIFICATION_LEGAL_BASIS_PLACEHOLDER)) {
return entry.getJustificationParagraph();
return (input) -> input.getEntry().getJustificationParagraph();
}
if (placeholder.equals(JUSTIFICATION_TEXT_PLACEHOLDER)) {
return entry.getJustificationReason();
return (input) -> input.getEntry().getJustificationReason();
}
if (placeholder.equals(EXCERPT_PLACEHOLDER)) {
return entry.getExcerpt();
return (input) -> input.getEntry().getExcerpt();
}
if (placeholder.equals(REDACTION_VALUE_PLACEHOLDER)) {
return entry.getValue() != null ? entry.getValue().replaceAll("\n", " ").replaceAll(" ", " ") : "";
return (input) -> input.getEntry().getValue() != null ? input.getEntry()
.getValue()
.replaceAll("\n", " ")
.replaceAll(" ", " ") : "";
}
throw new RuntimeException("invalid placeholder");
// TODO in case placeholder is invalid -> do not replace with empty string but throw previus exception
return (input) -> "";
}
@ -372,12 +345,12 @@ public class ExcelTemplateReportGenerationService {
if (dossierAttributesPlaceholders.containsKey(placeholder)) {
return dossierAttributesPlaceholders.get(placeholder);
}
throw new RuntimeException("unknown placeholder");
return null;
}
@SneakyThrows
public byte[] toByteArray(XSSFWorkbook workbook) {
public byte[] toByteArray(SXSSFWorkbook workbook) {
try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) {
workbook.write(byteArrayOutputStream);

109
redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/GeneratePlaceholderService.java Normal file
View File

@ -0,0 +1,109 @@
package com.iqser.red.service.redaction.report.v1.server.service;
import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.DOSSIER_NAME_PLACEHOLDER;
import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.FILE_NAME_PLACEHOLDER;
import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.FORMAT_DATE_ENG_PLACEHOLDER;
import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.FORMAT_DATE_GER_PLACEHOLDER;
import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.FORMAT_DATE_ISO_PLACEHOLDER;
import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.FORMAT_TIME_ISO_PLACEHOLDER;
import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.IUCLID_FUNCTION_PLACEHOLDER;
import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.SEEDS_FUNCTION_JUSTIFICATION_PLACEHOLDER;
import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.SEEDS_FUNCTION_REDACTION_GROUPED_BY_JUSTIFICATION_PAGES_PLACEHOLDER;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Base64;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DossierAttributeConfig;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.Dossier;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.DossierAttribute;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.DossierAttributeType;
import com.iqser.red.service.redaction.report.v1.server.client.DossierAttributesClient;
import com.iqser.red.service.redaction.report.v1.server.client.DossierAttributesConfigClient;
import com.iqser.red.service.redaction.report.v1.server.client.FileAttributesConfigClient;
import com.iqser.red.service.redaction.report.v1.server.model.ImagePlaceholder;
import com.iqser.red.service.redaction.report.v1.server.model.PlaceholderModel;
import lombok.RequiredArgsConstructor;
@Service
@RequiredArgsConstructor
public class GeneratePlaceholderService {
private final FileAttributesConfigClient fileAttributesClient;
private final DossierAttributesClient dossierAttributesClient;
private final DossierAttributesConfigClient dossierAttributesConfigClient;
public PlaceholderModel buildPlaceholders(Dossier dossier){
var dossierAttributes = dossierAttributesClient.getDossierAttributes(dossier.getId());
var dossierAttributesConfig = dossierAttributesConfigClient.getDossierAttributes(dossier.getDossierTemplateId());
var fileAttributePlaceHolders = getFileAttributePlaceholders(dossier.getDossierTemplateId());
List<String> placeholders = getDefaultPlaceholders();
List<ImagePlaceholder> imagePlaceholders = new ArrayList<>();
Map<String, String> dossierAttributesPlaceholder = new HashMap<>(); //KEY:placeholder, VALUE:value
for (DossierAttributeConfig attributeConfig : dossierAttributesConfig) {
for (DossierAttribute dossierAttribute : dossierAttributes) {
if (dossierAttribute.getDossierAttributeConfigId().equals(attributeConfig.getId())) {
if (attributeConfig.getType().equals(DossierAttributeType.IMAGE)) {
if (dossierAttribute.getValue() != null) {
if (dossierAttribute.getValue().startsWith("data:")) {
imagePlaceholders.add(new ImagePlaceholder(attributeConfig.getPlaceholder(), Base64.getDecoder()
.decode(dossierAttribute.getValue().split(",")[1])));
} else {
imagePlaceholders.add(new ImagePlaceholder(attributeConfig.getPlaceholder(), Base64.getDecoder().decode(dossierAttribute.getValue())));
}
}
} else {
if (dossierAttribute.getValue() == null) {
dossierAttributesPlaceholder.put(attributeConfig.getPlaceholder(), "");
} else {
dossierAttributesPlaceholder.put(attributeConfig.getPlaceholder(), dossierAttribute.getValue());
}
}
}
}
if (!dossierAttributesPlaceholder.containsKey(attributeConfig.getPlaceholder())) {
dossierAttributesPlaceholder.put(attributeConfig.getPlaceholder(), "");
}
}
placeholders.addAll(fileAttributePlaceHolders.keySet());
placeholders.addAll(dossierAttributesPlaceholder.keySet());
return new PlaceholderModel(placeholders,imagePlaceholders,dossierAttributesPlaceholder,fileAttributePlaceHolders);
}
private List<String> getDefaultPlaceholders() {
List<String> defPlaceholders = new ArrayList<>();
defPlaceholders.addAll(Arrays.asList(FILE_NAME_PLACEHOLDER, FORMAT_DATE_ISO_PLACEHOLDER, FORMAT_DATE_GER_PLACEHOLDER, FORMAT_DATE_ENG_PLACEHOLDER, FORMAT_TIME_ISO_PLACEHOLDER, DOSSIER_NAME_PLACEHOLDER, IUCLID_FUNCTION_PLACEHOLDER, SEEDS_FUNCTION_REDACTION_GROUPED_BY_JUSTIFICATION_PAGES_PLACEHOLDER, SEEDS_FUNCTION_JUSTIFICATION_PLACEHOLDER));
return defPlaceholders;
}
private Map<String, String> getFileAttributePlaceholders(String dossierTemplateId) {
Map<String, String> fileAttributePlaceholders = new HashMap<>();
var fileAttributesConfig = fileAttributesClient.getFileAttributeConfigs(dossierTemplateId);
fileAttributesConfig.forEach(fileAttributeConfig -> {
fileAttributePlaceholders.put(fileAttributeConfig.getPlaceholder(), fileAttributeConfig.getId());
});
return fileAttributePlaceholders;
}
}

126
redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ReportGenerationService.java
View File

@ -3,6 +3,7 @@ package com.iqser.red.service.redaction.report.v1.server.service;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.ReportTemplate;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.Dossier;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel;
import com.iqser.red.service.redaction.report.v1.api.model.ReportRequestMessage;
import com.iqser.red.service.redaction.report.v1.api.model.ReportType;
import com.iqser.red.service.redaction.report.v1.api.model.StoredFileInformation;
@ -10,29 +11,39 @@ import com.iqser.red.service.redaction.report.v1.server.client.DossierClient;
import com.iqser.red.service.redaction.report.v1.server.client.FileStatusClient;
import com.iqser.red.service.redaction.report.v1.server.client.RedactionLogClient;
import com.iqser.red.service.redaction.report.v1.server.client.ReportTemplateClient;
import com.iqser.red.service.redaction.report.v1.server.model.ExcelModel;
import com.iqser.red.service.redaction.report.v1.server.model.MultiFileDocument;
import com.iqser.red.service.redaction.report.v1.server.model.MultiFileWorkbook;
import com.iqser.red.service.redaction.report.v1.server.model.PlaceholderModel;
import com.iqser.red.service.redaction.report.v1.server.model.ReportRedactionEntry;
import com.iqser.red.service.redaction.report.v1.server.storage.ReportStorageService;
import com.iqser.red.service.redaction.v1.model.ManualChange;
import com.iqser.red.service.redaction.v1.model.RedactionLog;
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.apache.poi.xwpf.usermodel.XWPFDocument;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
@Slf4j
@Service
@RequiredArgsConstructor
@SuppressWarnings("PMD")
public class ReportGenerationService {
private final ReportStorageService reportStorageService;
@ -43,13 +54,15 @@ public class ReportGenerationService {
private final ReportTemplateClient reportTemplateClient;
private final RedactionLogClient redactionLogClient;
private final ExcelTemplateReportGenerationService excelTemplateReportGenerationService;
private final GeneratePlaceholderService generatePlaceholderService;
@SneakyThrows
public List<StoredFileInformation> generateReport(ReportRequestMessage reportMessage) {
List<StoredFileInformation> storedFileInformation = new ArrayList<>();
List<StoredFileInformation> storedFileInformation = Collections.synchronizedList(new ArrayList<>());
Dossier project = dossierClient.getDossierById(reportMessage.getDossierId(), true,false);
Dossier dossier = dossierClient.getDossierById(reportMessage.getDossierId(), true, false);
List<ReportTemplate> singleFilesTemplates = new ArrayList<>();
List<MultiFileWorkbook> multiFileWorkbooks = new ArrayList<>();
@ -58,11 +71,15 @@ public class ReportGenerationService {
try {
ReportTemplate reportTemplate = reportTemplateClient.getReportTemplate(reportMessage.getDossierTemplateId(), templateId);
if (reportTemplate.isMultiFileReport()) {
if(reportTemplate.getFileName().endsWith(".xlsx")) {
if (reportTemplate.getFileName().endsWith(".xlsx")) {
byte[] excelTemplate = reportStorageService.getReportTemplate(reportTemplate.getStorageId());
try (ByteArrayInputStream is = new ByteArrayInputStream(excelTemplate)) {
XSSFWorkbook workbook = new XSSFWorkbook(is);
MultiFileWorkbook multiFileWorkbook = new MultiFileWorkbook(workbook, templateId);
XSSFWorkbook readWorkbook = new XSSFWorkbook(is);
SXSSFWorkbook writeWorkbook = new SXSSFWorkbook();
for(Sheet sheet: readWorkbook){
writeWorkbook.createSheet(sheet.getSheetName());
}
MultiFileWorkbook multiFileWorkbook = new MultiFileWorkbook(readWorkbook, writeWorkbook, templateId, reportTemplate.getFileName());
multiFileWorkbooks.add(multiFileWorkbook);
} catch (IOException e) {
throw new RuntimeException("Could not generate multifile excel report.");
@ -71,7 +88,7 @@ public class ReportGenerationService {
byte[] wordTemplate = reportStorageService.getReportTemplate(reportTemplate.getStorageId());
try (ByteArrayInputStream is = new ByteArrayInputStream(wordTemplate)) {
XWPFDocument doc = new XWPFDocument(is);
MultiFileDocument multiFileDocument = new MultiFileDocument(doc, templateId);
MultiFileDocument multiFileDocument = new MultiFileDocument(doc, templateId, reportTemplate.getFileName());
multiFileDocuments.add(multiFileDocument);
} catch (IOException e) {
throw new RuntimeException("Could not generate multifile word report.");
@ -85,6 +102,9 @@ public class ReportGenerationService {
}
}
var placeholderModel = generatePlaceholderService.buildPlaceholders(dossier);
ExcelModel excelModel = null;
int i = 1;
for (int j = 0; j < reportMessage.getFileIds().size(); j++) {
@ -96,39 +116,25 @@ public class ReportGenerationService {
List<ReportRedactionEntry> reportEntries = getReportEntries(reportMessage.getDossierId(), reportMessage.getFileIds()
.get(j), fileStatus.isExcluded());
var isLastFile = j == reportMessage.getFileIds().size() - 1;
for (MultiFileWorkbook multiFileWorkbook : multiFileWorkbooks) {
excelTemplateReportGenerationService.generateReport(reportEntries, reportMessage.getDossierTemplateId(), multiFileWorkbook.getWorkbook(), fileStatus, project, j == reportMessage.getFileIds()
.size() - 1);
}
for (MultiFileDocument multiFileDocument : multiFileDocuments) {
wordReportGenerationService.generateReport(ReportType.WORD_TEMPLATE_MULTI_FILE, reportEntries, reportMessage.getDossierTemplateId(), multiFileDocument.getDocument(), fileStatus, project, j == reportMessage.getFileIds()
.size() - 1);
}
for (ReportTemplate reportTemplate : singleFilesTemplates) {
if (reportTemplate.getFileName().endsWith(".xlsx")) {
byte[] excelTemplate = reportStorageService.getReportTemplate(reportTemplate.getStorageId());
try (ByteArrayInputStream is = new ByteArrayInputStream(excelTemplate)) {
XSSFWorkbook workbook = new XSSFWorkbook(is);
excelTemplateReportGenerationService.generateReport(reportEntries, reportMessage.getDossierTemplateId(), workbook, fileStatus, project, true);
byte[] template = excelTemplateReportGenerationService.toByteArray(workbook);
String storageId = reportStorageService.storeObject(reportMessage.getDownloadId(), template);
storedFileInformation.add(new StoredFileInformation(reportMessage.getFileIds()
.get(j), storageId, ReportType.EXCEL_TEMPLATE_SINGLE_FILE, reportTemplate.getTemplateId()));
} catch (IOException e) {
throw new RuntimeException("Could not generate singlefile excel report.");
}
} else {
byte[] wordTemplate= reportStorageService.getReportTemplate(reportTemplate.getStorageId());
try (ByteArrayInputStream is = new ByteArrayInputStream(wordTemplate)) {
XWPFDocument doc = new XWPFDocument(is);
wordReportGenerationService.generateReport(ReportType.WORD_SINGLE_FILE, reportEntries, reportMessage.getDossierTemplateId(), doc, fileStatus, project, true);
byte[] template = wordReportGenerationService.toByteArray(doc);
String storageId = reportStorageService.storeObject(reportMessage.getDownloadId(), template);
storedFileInformation.add(new StoredFileInformation(reportMessage.getFileIds().get(j), storageId, ReportType.WORD_SINGLE_FILE, reportTemplate.getTemplateId()));
} catch (IOException e) {
throw new RuntimeException("Could not generate singlefile word report.");
}
if(excelModel == null){
excelModel = excelTemplateReportGenerationService.caculateExcelModel(multiFileWorkbook.getReadWorkBook().getSheetAt(0));
}
excelModel = excelTemplateReportGenerationService.generateReport(reportEntries, placeholderModel, multiFileWorkbook.getTemplateName(), multiFileWorkbook.getWriteWorkbook(), fileStatus, dossier, isLastFile, excelModel);
}
for (MultiFileDocument multiFileDocument : multiFileDocuments) {
wordReportGenerationService.generateReport(reportEntries, placeholderModel, multiFileDocument.getTemplateName(), multiFileDocument.getDocument(), fileStatus, dossier, isLastFile);
}
for (ReportTemplate reportTemplate : singleFilesTemplates) {
storedFileInformation.add(createReportFromTemplate(dossier, fileStatus, placeholderModel, reportTemplate.getFileName(), reportMessage.getDownloadId(), reportEntries, reportTemplate));
}
long end = System.currentTimeMillis();
@ -138,7 +144,7 @@ public class ReportGenerationService {
}
for (MultiFileWorkbook multiFileWorkbook : multiFileWorkbooks) {
byte[] template = excelTemplateReportGenerationService.toByteArray(multiFileWorkbook.getWorkbook());
byte[] template = excelTemplateReportGenerationService.toByteArray(multiFileWorkbook.getWriteWorkbook());
String storageId = reportStorageService.storeObject(reportMessage.getDownloadId(), template);
storedFileInformation.add(new StoredFileInformation(null, storageId, ReportType.EXCEL_TEMPLATE_MULTI_FILE, multiFileWorkbook.getTemplateId()));
}
@ -146,13 +152,49 @@ public class ReportGenerationService {
for (MultiFileDocument multiFileDocument : multiFileDocuments) {
byte[] template = wordReportGenerationService.toByteArray(multiFileDocument.getDocument());
String storageId = reportStorageService.storeObject(reportMessage.getDownloadId(), template);
storedFileInformation.add(new StoredFileInformation( null, storageId, ReportType.WORD_TEMPLATE_MULTI_FILE, multiFileDocument.getTemplateId()));
storedFileInformation.add(new StoredFileInformation(null, storageId, ReportType.WORD_TEMPLATE_MULTI_FILE, multiFileDocument.getTemplateId()));
}
return storedFileInformation;
}
private StoredFileInformation createReportFromTemplate(Dossier dossier, FileModel fileStatus,
PlaceholderModel placeholderModel, String templateName,
String downloadId, List<ReportRedactionEntry> reportEntries,
ReportTemplate reportTemplate) {
if (reportTemplate.getFileName().endsWith(".xlsx")) {
byte[] excelTemplate = reportStorageService.getReportTemplate(reportTemplate.getStorageId());
try (ByteArrayInputStream is = new ByteArrayInputStream(excelTemplate)) {
XSSFWorkbook readWorkbook = new XSSFWorkbook(is);
SXSSFWorkbook writeWorkbook = new SXSSFWorkbook();
for(Sheet sheet: readWorkbook){
writeWorkbook.createSheet(sheet.getSheetName());
}
var excelModel = excelTemplateReportGenerationService.caculateExcelModel(readWorkbook.getSheetAt(0));
excelTemplateReportGenerationService.generateReport(reportEntries, placeholderModel, templateName, writeWorkbook, fileStatus, dossier, true, excelModel);
byte[] template = excelTemplateReportGenerationService.toByteArray(writeWorkbook);
String storageId = reportStorageService.storeObject(downloadId, template);
return new StoredFileInformation(fileStatus.getId(), storageId, ReportType.EXCEL_TEMPLATE_SINGLE_FILE, reportTemplate.getTemplateId());
} catch (IOException e) {
throw new RuntimeException("Could not generate singlefile excel report.");
}
} else {
byte[] wordTemplate = reportStorageService.getReportTemplate(reportTemplate.getStorageId());
try (ByteArrayInputStream is = new ByteArrayInputStream(wordTemplate)) {
XWPFDocument doc = new XWPFDocument(is);
wordReportGenerationService.generateReport(reportEntries, placeholderModel, templateName, doc, fileStatus, dossier, true);
byte[] template = wordReportGenerationService.toByteArray(doc);
String storageId = reportStorageService.storeObject(downloadId, template);
return new StoredFileInformation(fileStatus.getId(), storageId, ReportType.WORD_SINGLE_FILE, reportTemplate.getTemplateId());
} catch (IOException e) {
throw new RuntimeException("Could not generate singlefile word report.");
}
}
}
private List<ReportRedactionEntry> getReportEntries(String dossierId, String fileId, boolean isExcluded) {
if (isExcluded) {
@ -165,10 +207,10 @@ public class ReportGenerationService {
//filter DECLINED redactions out, they should not be in reports
Iterator<RedactionLogEntry> iter = redactionLog.getRedactionLogEntry().iterator();
while(iter.hasNext()) {
while (iter.hasNext()) {
RedactionLogEntry redactionLogEntry = iter.next();
for(ManualChange manualChange : redactionLogEntry.getManualChanges()) {
if(manualChange.getAnnotationStatus().equals(AnnotationStatus.DECLINED)) {
for (ManualChange manualChange : redactionLogEntry.getManualChanges()) {
if (manualChange.getAnnotationStatus().equals(AnnotationStatus.DECLINED)) {
iter.remove();
}
}

359
redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/WordReportGenerationService.java
View File

@ -1,157 +1,101 @@
package com.iqser.red.service.redaction.report.v1.server.service;
import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.DOSSIER_NAME_PLACEHOLDER;
import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.EXCERPT_PLACEHOLDER;
import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.FILE_NAME_PLACEHOLDER;
import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.FORMAT_DATE_ENG;
import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.FORMAT_DATE_ENG_PLACEHOLDER;
import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.FORMAT_DATE_GER;
import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.FORMAT_DATE_GER_PLACEHOLDER;
import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.FORMAT_DATE_ISO;
import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.FORMAT_DATE_ISO_PLACEHOLDER;
import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.FORMAT_TIME_ISO;
import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.FORMAT_TIME_ISO_PLACEHOLDER;
import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.IUCLID_FUNCTION_PLACEHOLDER;
import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.JUSTIFICATION_LEGAL_BASIS_PLACEHOLDER;
import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.JUSTIFICATION_PARAGRAPH_PLACEHOLDER;
import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.JUSTIFICATION_PLACEHOLDER;
import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.JUSTIFICATION_REASON_PLACEHOLDER;
import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.JUSTIFICATION_TEXT_PLACEHOLDER;
import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.PAGE_PLACEHOLDER;
import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.PARAGRAPH_PLACEHOLDER;
import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.REDACTION_VALUE_PLACEHOLDER;
import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.SEEDS_FUNCTION_JUSTIFICATION_PLACEHOLDER;
import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.SEEDS_FUNCTION_REDACTION_GROUPED_BY_JUSTIFICATION_PAGES_PLACEHOLDER;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.Dossier;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel;
import com.iqser.red.service.redaction.report.v1.server.model.*;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.poi.openxml4j.exceptions.InvalidFormatException;
import org.apache.poi.util.Dimension2DDouble;
import org.apache.poi.util.Units;
import org.apache.poi.xwpf.usermodel.*;
import org.springframework.stereotype.Service;
import javax.imageio.ImageIO;
import java.awt.image.BufferedImage;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Base64;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
import java.util.*;
import java.util.function.Function;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import javax.imageio.ImageIO;
import org.apache.commons.lang3.StringUtils;
import org.apache.poi.openxml4j.exceptions.InvalidFormatException;
import org.apache.poi.util.Dimension2DDouble;
import org.apache.poi.util.Units;
import org.apache.poi.xwpf.usermodel.XWPFDocument;
import org.apache.poi.xwpf.usermodel.XWPFParagraph;
import org.apache.poi.xwpf.usermodel.XWPFRun;
import org.apache.poi.xwpf.usermodel.XWPFTable;
import org.apache.poi.xwpf.usermodel.XWPFTableCell;
import org.apache.poi.xwpf.usermodel.XWPFTableRow;
import org.openxmlformats.schemas.wordprocessingml.x2006.main.CTRow;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DossierAttributeConfig;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.ReportTemplate;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.Dossier;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.DossierAttribute;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.DossierAttributeType;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel;
import com.iqser.red.service.redaction.report.v1.api.model.ReportType;
import com.iqser.red.service.redaction.report.v1.server.client.DossierAttributesClient;
import com.iqser.red.service.redaction.report.v1.server.client.DossierAttributesConfigClient;
import com.iqser.red.service.redaction.report.v1.server.client.FileAttributesConfigClient;
import com.iqser.red.service.redaction.report.v1.server.model.ColoredText;
import com.iqser.red.service.redaction.report.v1.server.model.ImagePlaceholder;
import com.iqser.red.service.redaction.report.v1.server.model.ReportRedactionEntry;
import com.iqser.red.service.redaction.report.v1.server.storage.ReportStorageService;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.*;
@Slf4j
@Service
@RequiredArgsConstructor
@SuppressWarnings("PMD")
public class WordReportGenerationService {
private final FileAttributesConfigClient fileAttributesClient;
private final DossierAttributesClient dossierAttributesClient;
private final DossierAttributesConfigClient dossierAttributesConfigClient;
private final IuclidFunctionService iuclidFunctionService;
public XWPFDocument generateReport(ReportType reportType, List<ReportRedactionEntry> reportEntries, String dossierTemplateId, XWPFDocument doc, FileModel fileStatus,
Dossier dossier, boolean isLastFile) {
List<String> placeholders = getDefaultPlaceholders();
List<ImagePlaceholder> imagePlaceholders = new ArrayList<>();
var dossierAttributes = dossierAttributesClient.getDossierAttributes(dossier.getId());
var dossierAttributesConfig = dossierAttributesConfigClient.getDossierAttributes(dossierTemplateId);
Map<String, String> dossierAttributesPlaceholder = new HashMap<>(); //KEY:placeholder, VALUE:value
for (DossierAttributeConfig attributeConfig : dossierAttributesConfig) {
for (DossierAttribute dossierAttribute : dossierAttributes) {
if (dossierAttribute.getDossierAttributeConfigId().equals(attributeConfig.getId())) {
if (attributeConfig.getType().equals(DossierAttributeType.IMAGE)) {
if (dossierAttribute.getValue() != null) {
if (dossierAttribute.getValue().startsWith("data:")) {
imagePlaceholders.add(new ImagePlaceholder(attributeConfig.getPlaceholder(), Base64.getDecoder()
.decode(dossierAttribute.getValue().split(",")[1])));
} else {
imagePlaceholders.add(new ImagePlaceholder(attributeConfig.getPlaceholder(), Base64.getDecoder().decode(dossierAttribute.getValue())));
}
}
} else {
if (dossierAttribute.getValue() == null) {
dossierAttributesPlaceholder.put(attributeConfig.getPlaceholder(), "");
} else {
dossierAttributesPlaceholder.put(attributeConfig.getPlaceholder(), dossierAttribute.getValue());
}
}
}
}
if (!dossierAttributesPlaceholder.containsKey(attributeConfig.getPlaceholder())) {
dossierAttributesPlaceholder.put(attributeConfig.getPlaceholder(), "");
}
}
Map<String, String> fileAttributePlaceholders = getFileAttributePlaceholders(dossierTemplateId);
placeholders.addAll(fileAttributePlaceholders.keySet());
placeholders.addAll(dossierAttributesPlaceholder.keySet());
public XWPFDocument generateReport(List<ReportRedactionEntry> reportEntries,
PlaceholderModel placeholderModel,
String reportTemplateName,
XWPFDocument doc, FileModel fileStatus,
Dossier dossier, boolean isLastFile) {
long start = System.currentTimeMillis();
try {
for (ImagePlaceholder imagePlaceholder : imagePlaceholders) {
for (ImagePlaceholder imagePlaceholder : placeholderModel.getImagePlaceholders()) {
replaceImagePlaceholders(doc, imagePlaceholder);
}
long t1 = System.currentTimeMillis();
XWPFTable table = getRedactionTable(doc);
Map<Integer, String> placeholderCellPos = computePlaceholderPos(table);
addTableRows(table, reportEntries, fileStatus.getFilename(), fileStatus, fileAttributePlaceholders, placeholderCellPos);
for (String placeholder : placeholders) {
String placeholderValue = getPlaceholderValue(placeholder, dossier, fileStatus, fileAttributePlaceholders, dossierAttributesPlaceholder, reportEntries);
if (placeholderValue != null) {
replaceTextPlaceholders(doc, placeholder, placeholderValue);
}
}
if(!isLastFile) {
readdPlaceholders(table, placeholderCellPos);
var placeholderFunctions = computePlaceholderPos(table);
addTableRows(table, reportEntries, fileStatus.getFilename(), fileStatus, placeholderModel.getFileAttributePlaceHolders(), placeholderFunctions);
long t2 = System.currentTimeMillis();
log.debug("Table time: {}", (t2 - t1));
t1 = System.currentTimeMillis();
replaceTextPlaceholders(doc, placeholderModel, dossier, fileStatus, table, reportEntries);
if (isLastFile) {
removePlaceholdersRow(table);
}
t2 = System.currentTimeMillis();
log.debug("Text time: {}", (t2 - t1));
long end = System.currentTimeMillis();
log.info("Report Generation took: {} for file with id {}, pageCount: {}, redactionLogEntryCount: {}, reportName: {}, className: {}", end - start,
fileStatus.getId(), fileStatus.getNumberOfPages(), reportEntries.size(), reportTemplateName, getClass().getSimpleName());
return doc;
} catch (Exception e) {
log.error(e.getMessage() + " in file: " + fileStatus.getFilename());
throw new RuntimeException(e);
}
}
private void removePlaceholdersRow(XWPFTable table) {
if (table == null){
return;
}
for (int j = 0; j < table.getRows().size(); j++) {
for (int i = 0; i < table.getRows().get(j).getTableCells().size(); i++) {
XWPFTableCell cell = table.getRows().get(j).getTableCells().get(i);
if (containsRedactionPlaceholder(cell.getText())) {
table.removeRow(j);
return;
}
}
}
}
@ -168,26 +112,6 @@ public class WordReportGenerationService {
}
private List<String> getDefaultPlaceholders() {
List<String> defPlaceholders = new ArrayList<>();
defPlaceholders.addAll(Arrays.asList(FILE_NAME_PLACEHOLDER, FORMAT_DATE_ISO_PLACEHOLDER, FORMAT_DATE_GER_PLACEHOLDER, FORMAT_DATE_ENG_PLACEHOLDER, FORMAT_TIME_ISO_PLACEHOLDER, DOSSIER_NAME_PLACEHOLDER, IUCLID_FUNCTION_PLACEHOLDER, SEEDS_FUNCTION_REDACTION_GROUPED_BY_JUSTIFICATION_PAGES_PLACEHOLDER, SEEDS_FUNCTION_JUSTIFICATION_PLACEHOLDER));
return defPlaceholders;
}
private Map<String, String> getFileAttributePlaceholders(String dossierTemplateId) {
Map<String, String> fileAttributePlaceholders = new HashMap<>();
var fileAttributesConfig = fileAttributesClient.getFileAttributeConfigs(dossierTemplateId);
fileAttributesConfig.forEach(fileAttributeConfig -> {
fileAttributePlaceholders.put(fileAttributeConfig.getPlaceholder(), fileAttributeConfig.getId());
});
return fileAttributePlaceholders;
}
private String getPlaceholderValue(String placeholder, Dossier project, FileModel fileStatus, Map<String, String> fileAttributePlaceholders,
Map<String, String> dossierAttributesPlaceholders, List<ReportRedactionEntry> reportRedactionEntries) {
@ -251,53 +175,70 @@ public class WordReportGenerationService {
}
public void replaceTextPlaceholders(XWPFDocument doc, String search, String replace) {
public void replaceTextPlaceholders(XWPFDocument doc, PlaceholderModel placeholderModel, Dossier dossier, FileModel fileModel, XWPFTable tableToSkip, List<ReportRedactionEntry> reportRedactionEntries) {
replacePlaceholderInParagraph(doc.getParagraphs(), search, replace);
Map<String, String> placeHolderValueMap = new HashMap<>();
for (String placeholder : placeholderModel.getPlaceholders()) {
String placeholderValue = getPlaceholderValue(placeholder, dossier, fileModel, placeholderModel.getFileAttributePlaceHolders(), placeholderModel.getDossierAttributesPlaceholder(), reportRedactionEntries);
placeHolderValueMap.put(placeholder, placeholderValue);
}
replacePlaceholderInParagraph(doc.getParagraphs(), placeHolderValueMap);
for (XWPFTable tbl : doc.getTables()) {
if (tableToSkip == tbl) {
// already processed for redactionLog Entries
continue;
}
for (XWPFTableRow row : tbl.getRows()) {
for (XWPFTableCell cell : row.getTableCells()) {
replacePlaceholderInParagraph(cell.getParagraphs(), search, replace);
replacePlaceholderInParagraph(cell.getParagraphs(), placeHolderValueMap);
}
}
}
}
private void replacePlaceholderInParagraph(List<XWPFParagraph> paragraphs, String search, String replace) {
private void replacePlaceholderInParagraph(List<XWPFParagraph> paragraphs, Map<String, String> placeholderValueMap) {
for (XWPFParagraph p : paragraphs) {
String paragraphText = p.getText();
if (paragraphText.contains(search)) {
String escapedSearch = Pattern.quote(search);
String escapedReplace = Matcher.quoteReplacement(replace);
try {
paragraphText = paragraphText.replaceAll(escapedSearch, escapedReplace);
XWPFRun run = p.getRuns().get(0);
run.setText(paragraphText, 0);
} catch (Exception e) {
log.error("Could not replace {} with {}", escapedSearch, escapedReplace);
throw new RuntimeException(String.format("Could not replace %s with %s", escapedSearch, escapedReplace), e);
}
int size = p.getRuns().size();
for (int i = 1; i <= size; i++) {
// 0th run has to stay for placeholders without "\n", because it contains the value of the placeholder
p.removeRun(1);
}
if (paragraphText.contains("\n")) {
p.removeRun(0);
String[] stringsOnNewLines = paragraphText.split("\n");
for (int i = 0; i < stringsOnNewLines.length; i++) {
p.insertNewRun(i);
XWPFRun newRun = p.getRuns().get(i);
String textForLine = stringsOnNewLines[i];
ColoredText coloredText = getColor(textForLine);
newRun.setText(coloredText.getText());
if (coloredText.getColor() != null) {
newRun.setTextHighlightColor(coloredText.getColor());
}
newRun.addCarriageReturn();
for (var entry : placeholderValueMap.entrySet()) {
var search = entry.getKey();
var replace = entry.getValue();
if (paragraphText.contains(search)) {
String escapedSearch = Pattern.quote(search);
String escapedReplace = Matcher.quoteReplacement(replace);
try {
paragraphText = paragraphText.replaceAll(escapedSearch, escapedReplace);
XWPFRun run = p.getRuns().get(0);
run.setText(paragraphText, 0);
} catch (Exception e) {
log.error("Could not replace {} with {}", escapedSearch, escapedReplace);
throw new RuntimeException(String.format("Could not replace %s with %s", escapedSearch, escapedReplace), e);
}
int size = p.getRuns().size();
for (int i = 1; i <= size; i++) {
// 0th run has to stay for placeholders without "\n", because it contains the value of the placeholder
p.removeRun(1);
}
if (paragraphText.contains("\n")) {
p.removeRun(0);
String[] stringsOnNewLines = paragraphText.split("\n");
for (int i = 0; i < stringsOnNewLines.length; i++) {
p.insertNewRun(i);
XWPFRun newRun = p.getRuns().get(i);
String textForLine = stringsOnNewLines[i];
ColoredText coloredText = getColor(textForLine);
newRun.setText(coloredText.getText());
if (coloredText.getColor() != null) {
newRun.setTextHighlightColor(coloredText.getColor());
}
newRun.addCarriageReturn();
}
}
}
}
@ -323,28 +264,30 @@ public class WordReportGenerationService {
}
private Map<Integer, String> computePlaceholderPos(XWPFTable table) {
Map<Integer, String> placeholderCellPos = new HashMap<>();
private Map<Integer, Function<TextPlaceholderInput, String>> computePlaceholderPos(XWPFTable table) {
Map<Integer, Function<TextPlaceholderInput, String>> placeholderCellPos = new HashMap<>();
if(table != null) {
int placeholderRow = -1;
if (table != null) {
for (int j = 0; j < table.getRows().size(); j++) {
for (int i = 0; i < table.getRows().get(j).getTableCells().size(); i++) {
XWPFTableCell cell = table.getRows().get(j).getTableCells().get(i);
if (containsRedactionPlaceholder(cell.getText())) {
placeholderCellPos.put(i, cell.getText());
placeholderRow = j;
placeholderCellPos.put(i, getFunctionForPlaceHolder(cell.getText()));
} else if (cell.getText().isEmpty()) {
placeholderCellPos.put(i, "");
placeholderCellPos.put(i, (input) -> "");
}
}
}
table.removeRow(placeholderRow);
}
return placeholderCellPos;
}
private void addTableRows(XWPFTable table, List<ReportRedactionEntry> reportEntries, String filename, FileModel fileStatus, Map<String, String> fileAttributePlaceholders, Map<Integer, String> placeholderCellPos) {
private void addTableRows(XWPFTable table,
List<ReportRedactionEntry> reportEntries,
String filename,
FileModel fileStatus,
Map<String, String> fileAttributePlaceholders,
Map<Integer, Function<TextPlaceholderInput, String>> placeholderCellPos) {
if (table == null) {
return;
@ -354,23 +297,22 @@ public class WordReportGenerationService {
var redactionsPerJustification = getRedactionsPerJustification(reportEntries);
for (Map.Entry<String, List<ReportRedactionEntry>> entry : redactionsPerJustification.entrySet()) {
XWPFTableRow row = table.createRow();
for (Map.Entry<Integer, String> entry1 : placeholderCellPos.entrySet()) {
if (!entry1.getValue().isEmpty()) {
setText(row.getCell(entry1.getKey()), replaceSeedsPlaceholder(entry, filename, entry1.getValue(), fileStatus, fileAttributePlaceholders));
} else {
setText(row.getCell(entry1.getKey()), "");
}
}
// for (Map.Entry<Integer, Function<String, String>> entry1 : placeholderCellPos.entrySet()) {
//
// setText(row.getCell(entry1.getKey()), replaceSeedsPlaceholder(entry, filename, entry1.getValue(), fileStatus, fileAttributePlaceholders));
//
// if (!entry1.getValue().isEmpty()) {
//
// } else {
// setText(row.getCell(entry1.getKey()), "");
// }
// }
}
} else {
reportEntries.forEach(entry -> {
XWPFTableRow row = table.createRow();
for (Map.Entry<Integer, String> entry1 : placeholderCellPos.entrySet()) {
if (!entry1.getValue().isEmpty()) {
setText(row.getCell(entry1.getKey()), replaceTextPlaceholderWithEntries(entry, filename, entry1.getValue()));
} else {
setText(row.getCell(entry1.getKey()), "");
}
for (Map.Entry<Integer, Function<TextPlaceholderInput, String>> entry1 : placeholderCellPos.entrySet()) {
setText(row.getCell(entry1.getKey()), entry1.getValue().apply(new TextPlaceholderInput(filename, entry)));
}
});
}
@ -378,46 +320,47 @@ public class WordReportGenerationService {
private void readdPlaceholders(XWPFTable table, Map<Integer, String> placeholderCellPos) {
XWPFTableRow newRow = table.createRow();
for(int i = 0; i < table.getRow(0).getTableCells().size(); i++) {
for (int i = 0; i < table.getRow(0).getTableCells().size(); i++) {
String placeholder = placeholderCellPos.get(i);
setText(newRow.getCell(i), placeholder);
}
}
private String replaceTextPlaceholderWithEntries(ReportRedactionEntry entry, String filename, String placeholder) {
private Function<TextPlaceholderInput, String> getFunctionForPlaceHolder(String placeholder) {
if (placeholder.equals(FILE_NAME_PLACEHOLDER)) {
return filename;
return TextPlaceholderInput::getFilename;
}
if (placeholder.equals(PAGE_PLACEHOLDER)) {
return String.valueOf(entry.getPage());
return (input) -> String.valueOf(input.getEntry().getPage());
}
if (placeholder.equals(PARAGRAPH_PLACEHOLDER)) {
return entry.getSection();
return (input) -> input.getEntry().getSection();
}
if (placeholder.equals(JUSTIFICATION_PLACEHOLDER)) {
return entry.getJustification();
return (input) -> input.getEntry().getJustification();
}
if (placeholder.equals(JUSTIFICATION_PARAGRAPH_PLACEHOLDER)) {
return entry.getJustificationParagraph();
return (input) -> input.getEntry().getJustificationParagraph();
}
if (placeholder.equals(JUSTIFICATION_REASON_PLACEHOLDER)) {
return entry.getJustificationReason();
return (input) -> input.getEntry().getJustificationReason();
}
if (placeholder.equals(JUSTIFICATION_LEGAL_BASIS_PLACEHOLDER)) {
return entry.getJustificationParagraph();
return (input) -> input.getEntry().getJustificationParagraph();
}
if (placeholder.equals(JUSTIFICATION_TEXT_PLACEHOLDER)) {
return entry.getJustificationReason();
return (input) -> input.getEntry().getJustificationReason();
}
if (placeholder.equals(EXCERPT_PLACEHOLDER)) {
return entry.getExcerpt();
return (input) -> input.getEntry().getExcerpt();
}
if (placeholder.equals(REDACTION_VALUE_PLACEHOLDER)) {
return entry.getValue() != null ? entry.getValue().replaceAll("\n", " ").replaceAll(" ", " ") : "";
return (input) -> input.getEntry().getValue() != null ? input.getEntry().getValue().replaceAll("\n", " ").replaceAll(" ", " ") : "";
}
throw new RuntimeException("invalid placeholder");
// TODO in case placeholder is invalid -> do not replace with empty string but throw previus exception
return (input) -> "";
}

91
redaction-report-service-v1/redaction-report-service-server-v1/src/test/java/com/iqser/red/service/redaction/report/v1/server/RedactionReportIntegrationTest.java
View File

@ -12,7 +12,6 @@ import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.do
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileAttributeType;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.legalbasis.LegalBasis;
import com.iqser.red.service.redaction.report.v1.api.model.ReportType;
import com.iqser.red.service.redaction.report.v1.server.client.DossierAttributesClient;
import com.iqser.red.service.redaction.report.v1.server.client.DossierAttributesConfigClient;
import com.iqser.red.service.redaction.report.v1.server.client.FileAttributesConfigClient;
@ -20,6 +19,7 @@ import com.iqser.red.service.redaction.report.v1.server.client.ReportTemplateCli
import com.iqser.red.service.redaction.report.v1.server.configuration.MessagingConfiguration;
import com.iqser.red.service.redaction.report.v1.server.model.ReportRedactionEntry;
import com.iqser.red.service.redaction.report.v1.server.service.ExcelTemplateReportGenerationService;
import com.iqser.red.service.redaction.report.v1.server.service.GeneratePlaceholderService;
import com.iqser.red.service.redaction.report.v1.server.service.RedactionLogConverterService;
import com.iqser.red.service.redaction.report.v1.server.service.WordReportGenerationService;
import com.iqser.red.service.redaction.report.v1.server.storage.ReportStorageService;
@ -27,8 +27,10 @@ import com.iqser.red.service.redaction.v1.model.RedactionLog;
import com.iqser.red.storage.commons.service.StorageService;
import lombok.SneakyThrows;
import org.apache.commons.io.IOUtils;
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.apache.poi.xwpf.usermodel.XWPFDocument;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
@ -86,6 +88,9 @@ public class RedactionReportIntegrationTest {
@Autowired
private ExcelTemplateReportGenerationService excelTemplateReportGenerationService;
@Autowired
private GeneratePlaceholderService generatePlaceholderService;
@Test
public void testWordReportGeneration() throws IOException {
@ -159,7 +164,7 @@ public class RedactionReportIntegrationTest {
FileModel fileStatus = FileModel.builder().filename("FileABCD").fileAttributes(Map.of("3e9b9569-5d2e-4619-848b-dd0a3e96527f", "Test")).build();
Dossier project = Dossier.builder().id("dossierId").dossierName("projectName").build();
Dossier dossier = Dossier.builder().id("dossierId").dossierName("dossierName").build();
String templateId = "templateId";
String storageId = "storageId";
@ -170,8 +175,9 @@ public class RedactionReportIntegrationTest {
ClassPathResource templateResource = new ClassPathResource("templates/Seeds - New Justification Form.docx");
var placeholders = generatePlaceholderService.buildPlaceholders(dossier);
XWPFDocument doc = new XWPFDocument(templateResource.getInputStream());
wordReportGenerationService.generateReport(ReportType.WORD_SINGLE_FILE, reportEntries, dossierTemplateId, doc, fileStatus, project, true);
wordReportGenerationService.generateReport(reportEntries, placeholders, "test", doc, fileStatus, dossier, true);
byte[] report = wordReportGenerationService.toByteArray(doc);
try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/efsa_template_wrg.docx")) {
@ -180,6 +186,7 @@ public class RedactionReportIntegrationTest {
}
@Test
@Ignore
public void testExcelTemplateReportGeneration() throws IOException {
ClassPathResource redactionLogResource = new ClassPathResource("files/redactionLog.json");
@ -195,7 +202,10 @@ public class RedactionReportIntegrationTest {
String dossierId = "dossierId";
ClassPathResource templateResource = new ClassPathResource("templates/TestReport.xlsx");
XSSFWorkbook workbook = new XSSFWorkbook(templateResource.getInputStream());
XSSFWorkbook wb = new XSSFWorkbook(templateResource.getInputStream());
var excelModel = excelTemplateReportGenerationService.caculateExcelModel(wb.getSheetAt(0));
SXSSFWorkbook workbook = new SXSSFWorkbook();
workbook.createSheet("Test Row Peformance");
ClassPathResource legalBasisMappingResource = new ClassPathResource("files/legalBasisMapping.json");
List<LegalBasis> legalBasisMapping = objectMapper.readValue(legalBasisMappingResource.getInputStream(), new TypeReference<>() {
@ -226,8 +236,10 @@ public class RedactionReportIntegrationTest {
.storageId(storageId)
.build());
excelTemplateReportGenerationService.generateReport(reportEntries, "dossierTemplateId", workbook, fileModel, dossier, false);
excelTemplateReportGenerationService.generateReport(reportEntries2, "dossierTemplateId", workbook, fileModel2, dossier, true);
var placeholders = generatePlaceholderService.buildPlaceholders(dossier);
excelTemplateReportGenerationService.generateReport(reportEntries, placeholders, "test", workbook, fileModel, dossier, false, excelModel);
excelTemplateReportGenerationService.generateReport(reportEntries2, placeholders, "test", workbook, fileModel2, dossier, true, excelModel);
byte[] excelTemplateReport = excelTemplateReportGenerationService.toByteArray(workbook);
try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/test_report_excel_template.xlsx")) {
@ -281,7 +293,7 @@ public class RedactionReportIntegrationTest {
FileModel fileModel = FileModel.builder().filename("filename").fileAttributes(attributeIdToValue).build();
FileModel fileModel2 = FileModel.builder().filename("other file").fileAttributes(attributeIdToValue).build();
Dossier dossier = Dossier.builder().id("dossierId").dossierName("projectName").build();
Dossier dossier = Dossier.builder().id("dossierId").dossierName("dossierName").build();
when(reportTemplateClient.getReportTemplate(dossierTemplateId, templateId)).thenReturn(ReportTemplate.builder()
.dossierTemplateId(dossierTemplateId)
@ -290,36 +302,51 @@ public class RedactionReportIntegrationTest {
ClassPathResource wordTemplateResource = new ClassPathResource("templates/Justification Appendix A1.docx");
var placeholders = generatePlaceholderService.buildPlaceholders(dossier);
XWPFDocument doc = new XWPFDocument(wordTemplateResource.getInputStream());
wordReportGenerationService.generateReport(ReportType.WORD_SINGLE_FILE, reportEntries, dossierTemplateId, doc, fileModel, dossier, true);
wordReportGenerationService.generateReport(reportEntries, placeholders, "test", doc, fileModel, dossier, true);
byte[] wordReport = wordReportGenerationService.toByteArray(doc);
try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/efsa_template1.docx")) {
fileOutputStream.write(wordReport);
}
doc = new XWPFDocument(wordTemplateResource.getInputStream());
wordReportGenerationService.generateReport(ReportType.WORD_SINGLE_FILE, reportEntries2, dossierTemplateId, doc, fileModel2, dossier, true);
wordReportGenerationService.generateReport(reportEntries2, placeholders, "test", doc, fileModel2, dossier, true);
byte[] wordReport2 = wordReportGenerationService.toByteArray(doc);
try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/efsa_template2.docx")) {
fileOutputStream.write(wordReport2);
}
ClassPathResource excelTemplateResource = new ClassPathResource("templates/Excel Report.xlsx");
XSSFWorkbook workbook = new XSSFWorkbook(excelTemplateResource.getInputStream());
excelTemplateReportGenerationService.generateReport(reportEntries, "dossierTemplateId", workbook, fileModel, dossier, false);
excelTemplateReportGenerationService.generateReport(reportEntries2, "dossierTemplateId", workbook, fileModel2, dossier, true);
XSSFWorkbook wb = new XSSFWorkbook(excelTemplateResource.getInputStream());
var excelModel = excelTemplateReportGenerationService.caculateExcelModel(wb.getSheetAt(0));
SXSSFWorkbook workbook = new SXSSFWorkbook();
workbook.createSheet("Test Row Peformance");
excelTemplateReportGenerationService.generateReport(reportEntries, placeholders, "test", workbook, fileModel, dossier, false,excelModel);
excelTemplateReportGenerationService.generateReport(reportEntries2, placeholders, "test", workbook, fileModel2, dossier, true,excelModel);
byte[] excelTemplateReport = excelTemplateReportGenerationService.toByteArray(workbook);
try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/report_excel_template.xlsx")) {
fileOutputStream.write(excelTemplateReport);
}
XSSFWorkbook workbook2 = new XSSFWorkbook(excelTemplateResource.getInputStream());
excelTemplateReportGenerationService.generateReport(reportEntries, "dossierTemplateId", workbook2, fileModel, dossier, true);
XSSFWorkbook wb2 = new XSSFWorkbook(excelTemplateResource.getInputStream());
var excelModel2 = excelTemplateReportGenerationService.caculateExcelModel(wb2.getSheetAt(0));
SXSSFWorkbook workbook2 = new SXSSFWorkbook();
workbook2.createSheet("Test Row Peformance");
excelTemplateReportGenerationService.generateReport(reportEntries, placeholders, "test", workbook2, fileModel, dossier, true,excelModel2);
byte[] excelTemplateReport2 = excelTemplateReportGenerationService.toByteArray(workbook2);
try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/report_excel_template2.xlsx")) {
fileOutputStream.write(excelTemplateReport2);
}
XSSFWorkbook workbook3 = new XSSFWorkbook(excelTemplateResource.getInputStream());
excelTemplateReportGenerationService.generateReport(reportEntries2, "dossierTemplateId", workbook3, fileModel2, dossier, true);
XSSFWorkbook wb3 = new XSSFWorkbook(excelTemplateResource.getInputStream());
var excelModel3 = excelTemplateReportGenerationService.caculateExcelModel(wb2.getSheetAt(0));
SXSSFWorkbook workbook3 = new SXSSFWorkbook();
workbook3.createSheet("Test Row Peformance");
excelTemplateReportGenerationService.generateReport(reportEntries2, placeholders, "test", workbook3, fileModel2, dossier, true,excelModel3);
byte[] excelTemplateReport3 = excelTemplateReportGenerationService.toByteArray(workbook3);
try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/report_excel_template3.xlsx")) {
fileOutputStream.write(excelTemplateReport3);
@ -368,7 +395,7 @@ public class RedactionReportIntegrationTest {
attributeIdToValue.put("3e9b9569-5d2e-4619-848b-dd0a3e96527f", "Test");
FileModel fileModel = FileModel.builder().filename("filename").fileAttributes(attributeIdToValue).build();
Dossier project = Dossier.builder().id("dossierId").dossierName("projectName").build();
Dossier dossier = Dossier.builder().id("dossierId").dossierName("dossierName").build();
String templateId = "templateId";
String storageId = "storageId";
@ -378,9 +405,9 @@ public class RedactionReportIntegrationTest {
.build());
ClassPathResource templateResource = new ClassPathResource("templates/6464 appendix_b EFSA dRAR justification.docx");
var placeholders = generatePlaceholderService.buildPlaceholders(dossier);
XWPFDocument doc = new XWPFDocument(templateResource.getInputStream());
wordReportGenerationService.generateReport(ReportType.WORD_SINGLE_FILE, reportEntries, dossierTemplateId, doc, fileModel, project, true);
wordReportGenerationService.generateReport(reportEntries, placeholders, "test", doc, fileModel, dossier, true);
byte[] report = wordReportGenerationService.toByteArray(doc);
try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/efsa_template13.docx")) {
@ -416,7 +443,7 @@ public class RedactionReportIntegrationTest {
FileModel fileStatus = FileModel.builder().filename("VV123456").build();
Dossier project = Dossier.builder().id("dossierId").dossierName("projectName").build();
Dossier dossier = Dossier.builder().id("dossierId").dossierName("dossierName").build();
String templateId = "templateId";
String storageId = "storageId";
@ -434,9 +461,9 @@ public class RedactionReportIntegrationTest {
.storageId("storageId")
.uploadDate(OffsetDateTime.now())
.build();
var placeholders = generatePlaceholderService.buildPlaceholders(dossier);
XWPFDocument doc = new XWPFDocument(templateResource.getInputStream());
wordReportGenerationService.generateReport(ReportType.WORD_SINGLE_FILE, reportEntries, dossierTemplateId, doc, fileStatus, project, true);
wordReportGenerationService.generateReport(reportEntries, placeholders, "test", doc, fileStatus, dossier, true);
byte[] report = wordReportGenerationService.toByteArray(doc);
try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/iuclid_report_2.docx")) {
@ -490,7 +517,7 @@ public class RedactionReportIntegrationTest {
FileModel fileModel = FileModel.builder().filename("filename").fileAttributes(attributeIdToValue).build();
FileModel fileModel2 = FileModel.builder().filename("other file").fileAttributes(attributeIdToValue).build();
Dossier dossier = Dossier.builder().id("dossierId").dossierName("projectName").build();
Dossier dossier = Dossier.builder().id("dossierId").dossierName("dossierName").build();
when(reportTemplateClient.getReportTemplate(dossierTemplateId, templateId)).thenReturn(ReportTemplate.builder()
.dossierTemplateId(dossierTemplateId)
@ -498,9 +525,13 @@ public class RedactionReportIntegrationTest {
.build());
ClassPathResource excelTemplateResource = new ClassPathResource("templates/Excel Report.xlsx");
XSSFWorkbook workbook = new XSSFWorkbook(excelTemplateResource.getInputStream());
excelTemplateReportGenerationService.generateReport(emptyReportEntries, "dossierTemplateId", workbook, fileModel, dossier, false);
excelTemplateReportGenerationService.generateReport(reportEntries2, "dossierTemplateId", workbook, fileModel2, dossier, true);
XSSFWorkbook wb = new XSSFWorkbook(excelTemplateResource.getInputStream());
SXSSFWorkbook workbook = new SXSSFWorkbook();
workbook.createSheet("Test Row Peformance");
var placeholders = generatePlaceholderService.buildPlaceholders(dossier);
var excelModel = excelTemplateReportGenerationService.caculateExcelModel(wb.getSheetAt(0));
excelTemplateReportGenerationService.generateReport(emptyReportEntries, placeholders, "test", workbook, fileModel, dossier, false,excelModel);
excelTemplateReportGenerationService.generateReport(reportEntries2, placeholders, "test", workbook, fileModel2, dossier, true,excelModel);
byte[] excelTemplateReport = excelTemplateReportGenerationService.toByteArray(workbook);
try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/report_excel_templateAAA.xlsx")) {
fileOutputStream.write(excelTemplateReport);
@ -533,7 +564,7 @@ public class RedactionReportIntegrationTest {
FileModel fileStatus = FileModel.builder().filename("VV123456").build();
FileModel fileStatus2 = FileModel.builder().filename("second file").build();
Dossier project = Dossier.builder().id("dossierId").dossierName("projectName").build();
Dossier dossier = Dossier.builder().id("dossierId").dossierName("dossierName").build();
String templateId = "templateId";
String storageId = "storageId";
@ -545,8 +576,10 @@ public class RedactionReportIntegrationTest {
ClassPathResource templateResource = new ClassPathResource("templates/Seeds-NewJustificationForm.docx");
when(reportStorageService.getReportTemplate(storageId)).thenReturn(IOUtils.toByteArray(templateResource.getInputStream()));
XWPFDocument doc = new XWPFDocument(templateResource.getInputStream());
doc = wordReportGenerationService.generateReport(ReportType.WORD_TEMPLATE_MULTI_FILE, reportEntries, dossierTemplateId, doc, fileStatus, project, false);
doc = wordReportGenerationService.generateReport(ReportType.WORD_TEMPLATE_MULTI_FILE, reportEntries2, dossierTemplateId, doc, fileStatus2, project, true);
var placeholders = generatePlaceholderService.buildPlaceholders(dossier);
doc = wordReportGenerationService.generateReport(reportEntries, placeholders, "test", doc, fileStatus, dossier, false);
doc = wordReportGenerationService.generateReport(reportEntries2, placeholders, "test", doc, fileStatus2, dossier, true);
byte[] report = wordReportGenerationService.toByteArray(doc);
try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/seedsReport.docx")) {

132
redaction-report-service-v1/redaction-report-service-server-v1/src/test/java/com/iqser/red/service/redaction/report/v1/server/realdata/RealDataServiceTest.java Normal file
View File

@ -0,0 +1,132 @@
package com.iqser.red.service.redaction.report.v1.server.realdata;
import com.amazonaws.services.s3.AmazonS3;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DossierAttributeConfig;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.ReportTemplate;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.Dossier;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.DossierAttribute;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.DossierAttributeType;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileAttributeConfig;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileAttributeType;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.legalbasis.LegalBasis;
import com.iqser.red.service.redaction.report.v1.server.client.DossierAttributesClient;
import com.iqser.red.service.redaction.report.v1.server.client.DossierAttributesConfigClient;
import com.iqser.red.service.redaction.report.v1.server.client.FileAttributesConfigClient;
import com.iqser.red.service.redaction.report.v1.server.client.ReportTemplateClient;
import com.iqser.red.service.redaction.report.v1.server.configuration.MessagingConfiguration;
import com.iqser.red.service.redaction.report.v1.server.model.ReportRedactionEntry;
import com.iqser.red.service.redaction.report.v1.server.service.ExcelTemplateReportGenerationService;
import com.iqser.red.service.redaction.report.v1.server.service.GeneratePlaceholderService;
import com.iqser.red.service.redaction.report.v1.server.service.RedactionLogConverterService;
import com.iqser.red.service.redaction.report.v1.server.service.WordReportGenerationService;
import com.iqser.red.service.redaction.report.v1.server.storage.ReportStorageService;
import com.iqser.red.service.redaction.v1.model.RedactionLog;
import com.iqser.red.storage.commons.service.StorageService;
import org.apache.commons.io.IOUtils;
import org.apache.poi.xwpf.usermodel.XWPFDocument;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.core.io.ClassPathResource;
import org.springframework.test.context.junit4.SpringRunner;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.Arrays;
import java.util.Base64;
import java.util.List;
import java.util.Map;
import static com.iqser.red.service.redaction.report.v1.server.utils.OsUtils.getTemporaryDirectory;
import static org.mockito.Mockito.when;
import static org.springframework.boot.test.context.SpringBootTest.WebEnvironment.RANDOM_PORT;
@RunWith(SpringRunner.class)
@SpringBootTest(webEnvironment = RANDOM_PORT)
public class RealDataServiceTest {
@Autowired
private ObjectMapper objectMapper;
@MockBean
private StorageService storageService;
@MockBean
private ReportStorageService reportStorageService;
@MockBean
private ReportTemplateClient reportTemplateClient;
@MockBean
private FileAttributesConfigClient fileAttributesConfigClient;
@MockBean
private AmazonS3 s3Client;
@Autowired
private WordReportGenerationService wordReportGenerationService;
@MockBean
private MessagingConfiguration messagingConfiguration;
@Autowired
private RedactionLogConverterService redactionLogConverterService;
@MockBean
private DossierAttributesConfigClient dossierAttributesConfigClient;
@MockBean
private DossierAttributesClient dossierAttributesClient;
@Autowired
private ExcelTemplateReportGenerationService excelTemplateReportGenerationService;
@Autowired
private GeneratePlaceholderService generatePlaceholderService;
@Test
public void testWordReportGeneration() throws IOException {
String dossierTemplateId = "dossierTemplateId";
ClassPathResource redactionLogResource = new ClassPathResource("realdata/redaction-log.json");
RedactionLog redactionLog = objectMapper.readValue(redactionLogResource.getInputStream(), RedactionLog.class);
ClassPathResource legalBasisMappingResource = new ClassPathResource("files/legalBasisMapping.json");
List<LegalBasis> legalBasisMapping = objectMapper.readValue(legalBasisMappingResource.getInputStream(), new TypeReference<>() {
});
List<ReportRedactionEntry> reportEntries = redactionLogConverterService.convertAndSort(redactionLog, legalBasisMapping);
FileModel fileStatus = FileModel.builder().filename("FileABCD").fileAttributes(Map.of("3e9b9569-5d2e-4619-848b-dd0a3e96527f", "Test")).build();
Dossier dossier = Dossier.builder().id("dossierId").dossierName("dossierName").build();
String templateId = "templateId";
String storageId = "storageId";
when(reportTemplateClient.getReportTemplate(dossierTemplateId, templateId)).thenReturn(ReportTemplate.builder()
.dossierTemplateId(dossierTemplateId)
.storageId(storageId)
.build());
ClassPathResource templateResource = new ClassPathResource("realdata/Justification Appendix A2.docx");
var placeholders = generatePlaceholderService.buildPlaceholders(dossier);
XWPFDocument doc = new XWPFDocument(templateResource.getInputStream());
wordReportGenerationService.generateReport(reportEntries, placeholders, "test", doc, fileStatus, dossier, true);
byte[] report = wordReportGenerationService.toByteArray(doc);
try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/efsa_template_wrg.docx")) {
fileOutputStream.write(report);
}
}
}

BIN
redaction-report-service-v1/redaction-report-service-server-v1/src/test/resources/realdata/Justification Appendix A2.docx Normal file
View File

Binary file not shown.

1
redaction-report-service-v1/redaction-report-service-server-v1/src/test/resources/realdata/redaction-log.json Normal file
View File

File diff suppressed because one or more lines are too long

BIN
redaction-report-service-v1/redaction-report-service-server-v1/src/test/resources/templates/Excel Report.xlsx
View File

Binary file not shown.