RED-4036 - improved report service performance, added some logs

This commit is contained in:
Timo Bejan 2022-05-13 15:35:54 +03:00
parent cb8e044d76
commit 81febade14
6 changed files with 269 additions and 81 deletions

View File

@ -0,0 +1,9 @@
package com.iqser.red.service.redaction.report.v1.server.model;
import java.util.List;
import java.util.Map;
public interface PlaceholderInput {
}

View File

@ -0,0 +1,13 @@
package com.iqser.red.service.redaction.report.v1.server.model;
import lombok.AllArgsConstructor;
import lombok.Data;
@Data
@AllArgsConstructor
public class TextPlaceholderInput implements PlaceholderInput {
private String filename;
private ReportRedactionEntry entry;
}

View File

@ -1,12 +1,8 @@
package com.iqser.red.service.redaction.report.v1.server.service;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.ReportTemplate;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.Dossier;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel;
import com.iqser.red.service.redaction.report.v1.server.model.ColoredText;
import com.iqser.red.service.redaction.report.v1.server.model.ImagePlaceholder;
import com.iqser.red.service.redaction.report.v1.server.model.PlaceholderModel;
import com.iqser.red.service.redaction.report.v1.server.model.ReportRedactionEntry;
import com.iqser.red.service.redaction.report.v1.server.model.*;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
@ -24,6 +20,7 @@ import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.time.OffsetDateTime;
import java.util.*;
import java.util.function.Function;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
@ -52,19 +49,24 @@ public class WordReportGenerationService {
for (ImagePlaceholder imagePlaceholder : placeholderModel.getImagePlaceholders()) {
replaceImagePlaceholders(doc, imagePlaceholder);
}
long t1 = System.currentTimeMillis();
XWPFTable table = getRedactionTable(doc);
Map<Integer, String> placeholderCellPos = computePlaceholderPos(table);
addTableRows(table, reportEntries, fileStatus.getFilename(), fileStatus, placeholderModel.getFileAttributePlaceHolders(), placeholderCellPos);
for (String placeholder : placeholderModel.getPlaceholders()) {
String placeholderValue = getPlaceholderValue(placeholder, dossier, fileStatus, placeholderModel.getFileAttributePlaceHolders(), placeholderModel.getDossierAttributesPlaceholder(), reportEntries);
if (placeholderValue != null) {
replaceTextPlaceholders(doc, placeholder, placeholderValue);
}
}
if (!isLastFile) {
readdPlaceholders(table, placeholderCellPos);
var placeholderFunctions = computePlaceholderPos(table);
addTableRows(table, reportEntries, fileStatus.getFilename(), fileStatus, placeholderModel.getFileAttributePlaceHolders(), placeholderFunctions);
long t2 = System.currentTimeMillis();
log.warn("Table time: {}", (t2 - t1));
t1 = System.currentTimeMillis();
replaceTextPlaceholders(doc, placeholderModel, dossier, fileStatus, table);
if (isLastFile) {
removePlaceholdersRow(table);
}
t2 = System.currentTimeMillis();
log.warn("Text time: {}", (t2 - t1));
long end = System.currentTimeMillis();
@ -80,6 +82,18 @@ public class WordReportGenerationService {
}
private void removePlaceholdersRow(XWPFTable table) {
for (int j = 0; j < table.getRows().size(); j++) {
for (int i = 0; i < table.getRows().get(j).getTableCells().size(); i++) {
XWPFTableCell cell = table.getRows().get(j).getTableCells().get(i);
if (containsRedactionPlaceholder(cell.getText())) {
table.removeRow(j);
return;
}
}
}
}
private void replaceImagePlaceholders(XWPFDocument doc, ImagePlaceholder imagePlaceholder) {
@ -157,53 +171,70 @@ public class WordReportGenerationService {
}
public void replaceTextPlaceholders(XWPFDocument doc, String search, String replace) {
public void replaceTextPlaceholders(XWPFDocument doc, PlaceholderModel placeholderModel, Dossier dossier, FileModel fileModel, XWPFTable tableToSkip) {
replacePlaceholderInParagraph(doc.getParagraphs(), search, replace);
Map<String, String> placeHolderValueMap = new HashMap<>();
for (String placeholder : placeholderModel.getPlaceholders()) {
String placeholderValue = getPlaceholderValue(placeholder, dossier, fileModel, placeholderModel.getFileAttributePlaceHolders(), placeholderModel.getDossierAttributesPlaceholder(), new ArrayList<>());
placeHolderValueMap.put(placeholder, placeholderValue);
}
replacePlaceholderInParagraph(doc.getParagraphs(), placeHolderValueMap);
for (XWPFTable tbl : doc.getTables()) {
if (tableToSkip == tbl) {
// already processed for redactionLog Entries
continue;
}
for (XWPFTableRow row : tbl.getRows()) {
for (XWPFTableCell cell : row.getTableCells()) {
replacePlaceholderInParagraph(cell.getParagraphs(), search, replace);
replacePlaceholderInParagraph(cell.getParagraphs(), placeHolderValueMap);
}
}
}
}
private void replacePlaceholderInParagraph(List<XWPFParagraph> paragraphs, String search, String replace) {
private void replacePlaceholderInParagraph(List<XWPFParagraph> paragraphs, Map<String, String> placeholderValueMap) {
for (XWPFParagraph p : paragraphs) {
String paragraphText = p.getText();
if (paragraphText.contains(search)) {
String escapedSearch = Pattern.quote(search);
String escapedReplace = Matcher.quoteReplacement(replace);
try {
paragraphText = paragraphText.replaceAll(escapedSearch, escapedReplace);
XWPFRun run = p.getRuns().get(0);
run.setText(paragraphText, 0);
} catch (Exception e) {
log.error("Could not replace {} with {}", escapedSearch, escapedReplace);
throw new RuntimeException(String.format("Could not replace %s with %s", escapedSearch, escapedReplace), e);
}
int size = p.getRuns().size();
for (int i = 1; i <= size; i++) {
// 0th run has to stay for placeholders without "\n", because it contains the value of the placeholder
p.removeRun(1);
}
if (paragraphText.contains("\n")) {
p.removeRun(0);
String[] stringsOnNewLines = paragraphText.split("\n");
for (int i = 0; i < stringsOnNewLines.length; i++) {
p.insertNewRun(i);
XWPFRun newRun = p.getRuns().get(i);
String textForLine = stringsOnNewLines[i];
ColoredText coloredText = getColor(textForLine);
newRun.setText(coloredText.getText());
if (coloredText.getColor() != null) {
newRun.setTextHighlightColor(coloredText.getColor());
}
newRun.addCarriageReturn();
for (var entry : placeholderValueMap.entrySet()) {
var search = entry.getKey();
var replace = entry.getValue();
if (paragraphText.contains(search)) {
String escapedSearch = Pattern.quote(search);
String escapedReplace = Matcher.quoteReplacement(replace);
try {
paragraphText = paragraphText.replaceAll(escapedSearch, escapedReplace);
XWPFRun run = p.getRuns().get(0);
run.setText(paragraphText, 0);
} catch (Exception e) {
log.error("Could not replace {} with {}", escapedSearch, escapedReplace);
throw new RuntimeException(String.format("Could not replace %s with %s", escapedSearch, escapedReplace), e);
}
int size = p.getRuns().size();
for (int i = 1; i <= size; i++) {
// 0th run has to stay for placeholders without "\n", because it contains the value of the placeholder
p.removeRun(1);
}
if (paragraphText.contains("\n")) {
p.removeRun(0);
String[] stringsOnNewLines = paragraphText.split("\n");
for (int i = 0; i < stringsOnNewLines.length; i++) {
p.insertNewRun(i);
XWPFRun newRun = p.getRuns().get(i);
String textForLine = stringsOnNewLines[i];
ColoredText coloredText = getColor(textForLine);
newRun.setText(coloredText.getText());
if (coloredText.getColor() != null) {
newRun.setTextHighlightColor(coloredText.getColor());
}
newRun.addCarriageReturn();
}
}
}
}
@ -229,28 +260,30 @@ public class WordReportGenerationService {
}
private Map<Integer, String> computePlaceholderPos(XWPFTable table) {
Map<Integer, String> placeholderCellPos = new HashMap<>();
private Map<Integer, Function<TextPlaceholderInput, String>> computePlaceholderPos(XWPFTable table) {
Map<Integer, Function<TextPlaceholderInput, String>> placeholderCellPos = new HashMap<>();
if (table != null) {
int placeholderRow = -1;
for (int j = 0; j < table.getRows().size(); j++) {
for (int i = 0; i < table.getRows().get(j).getTableCells().size(); i++) {
XWPFTableCell cell = table.getRows().get(j).getTableCells().get(i);
if (containsRedactionPlaceholder(cell.getText())) {
placeholderCellPos.put(i, cell.getText());
placeholderRow = j;
placeholderCellPos.put(i, getFunctionForPlaceHolder(cell.getText()));
} else if (cell.getText().isEmpty()) {
placeholderCellPos.put(i, "");
placeholderCellPos.put(i, (input) -> "");
}
}
}
table.removeRow(placeholderRow);
}
return placeholderCellPos;
}
private void addTableRows(XWPFTable table, List<ReportRedactionEntry> reportEntries, String filename, FileModel fileStatus, Map<String, String> fileAttributePlaceholders, Map<Integer, String> placeholderCellPos) {
private void addTableRows(XWPFTable table,
List<ReportRedactionEntry> reportEntries,
String filename,
FileModel fileStatus,
Map<String, String> fileAttributePlaceholders,
Map<Integer, Function<TextPlaceholderInput, String>> placeholderCellPos) {
if (table == null) {
return;
@ -260,23 +293,22 @@ public class WordReportGenerationService {
var redactionsPerJustification = getRedactionsPerJustification(reportEntries);
for (Map.Entry<String, List<ReportRedactionEntry>> entry : redactionsPerJustification.entrySet()) {
XWPFTableRow row = table.createRow();
for (Map.Entry<Integer, String> entry1 : placeholderCellPos.entrySet()) {
if (!entry1.getValue().isEmpty()) {
setText(row.getCell(entry1.getKey()), replaceSeedsPlaceholder(entry, filename, entry1.getValue(), fileStatus, fileAttributePlaceholders));
} else {
setText(row.getCell(entry1.getKey()), "");
}
}
// for (Map.Entry<Integer, Function<String, String>> entry1 : placeholderCellPos.entrySet()) {
//
// setText(row.getCell(entry1.getKey()), replaceSeedsPlaceholder(entry, filename, entry1.getValue(), fileStatus, fileAttributePlaceholders));
//
// if (!entry1.getValue().isEmpty()) {
//
// } else {
// setText(row.getCell(entry1.getKey()), "");
// }
// }
}
} else {
reportEntries.forEach(entry -> {
XWPFTableRow row = table.createRow();
for (Map.Entry<Integer, String> entry1 : placeholderCellPos.entrySet()) {
if (!entry1.getValue().isEmpty()) {
setText(row.getCell(entry1.getKey()), replaceTextPlaceholderWithEntries(entry, filename, entry1.getValue()));
} else {
setText(row.getCell(entry1.getKey()), "");
}
for (Map.Entry<Integer, Function<TextPlaceholderInput, String>> entry1 : placeholderCellPos.entrySet()) {
setText(row.getCell(entry1.getKey()), entry1.getValue().apply(new TextPlaceholderInput(filename, entry)));
}
});
}
@ -291,39 +323,40 @@ public class WordReportGenerationService {
}
private String replaceTextPlaceholderWithEntries(ReportRedactionEntry entry, String filename, String placeholder) {
private Function<TextPlaceholderInput, String> getFunctionForPlaceHolder(String placeholder) {
if (placeholder.equals(FILE_NAME_PLACEHOLDER)) {
return filename;
return TextPlaceholderInput::getFilename;
}
if (placeholder.equals(PAGE_PLACEHOLDER)) {
return String.valueOf(entry.getPage());
return (input) -> String.valueOf(input.getEntry().getPage());
}
if (placeholder.equals(PARAGRAPH_PLACEHOLDER)) {
return entry.getSection();
return (input) -> input.getEntry().getSection();
}
if (placeholder.equals(JUSTIFICATION_PLACEHOLDER)) {
return entry.getJustification();
return (input) -> input.getEntry().getJustification();
}
if (placeholder.equals(JUSTIFICATION_PARAGRAPH_PLACEHOLDER)) {
return entry.getJustificationParagraph();
return (input) -> input.getEntry().getJustificationParagraph();
}
if (placeholder.equals(JUSTIFICATION_REASON_PLACEHOLDER)) {
return entry.getJustificationReason();
return (input) -> input.getEntry().getJustificationReason();
}
if (placeholder.equals(JUSTIFICATION_LEGAL_BASIS_PLACEHOLDER)) {
return entry.getJustificationParagraph();
return (input) -> input.getEntry().getJustificationParagraph();
}
if (placeholder.equals(JUSTIFICATION_TEXT_PLACEHOLDER)) {
return entry.getJustificationReason();
return (input) -> input.getEntry().getJustificationReason();
}
if (placeholder.equals(EXCERPT_PLACEHOLDER)) {
return entry.getExcerpt();
return (input) -> input.getEntry().getExcerpt();
}
if (placeholder.equals(REDACTION_VALUE_PLACEHOLDER)) {
return entry.getValue() != null ? entry.getValue().replaceAll("\n", " ").replaceAll(" ", " ") : "";
return (input) -> input.getEntry().getValue() != null ? input.getEntry().getValue().replaceAll("\n", " ").replaceAll(" ", " ") : "";
}
throw new RuntimeException("invalid placeholder");
// TODO in case placeholder is invalid -> do not replace with empty string but throw previus exception
return (input) -> "";
}

View File

@ -0,0 +1,132 @@
package com.iqser.red.service.redaction.report.v1.server.realdata;
import com.amazonaws.services.s3.AmazonS3;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DossierAttributeConfig;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.ReportTemplate;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.Dossier;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.DossierAttribute;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.DossierAttributeType;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileAttributeConfig;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileAttributeType;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.legalbasis.LegalBasis;
import com.iqser.red.service.redaction.report.v1.server.client.DossierAttributesClient;
import com.iqser.red.service.redaction.report.v1.server.client.DossierAttributesConfigClient;
import com.iqser.red.service.redaction.report.v1.server.client.FileAttributesConfigClient;
import com.iqser.red.service.redaction.report.v1.server.client.ReportTemplateClient;
import com.iqser.red.service.redaction.report.v1.server.configuration.MessagingConfiguration;
import com.iqser.red.service.redaction.report.v1.server.model.ReportRedactionEntry;
import com.iqser.red.service.redaction.report.v1.server.service.ExcelTemplateReportGenerationService;
import com.iqser.red.service.redaction.report.v1.server.service.GeneratePlaceholderService;
import com.iqser.red.service.redaction.report.v1.server.service.RedactionLogConverterService;
import com.iqser.red.service.redaction.report.v1.server.service.WordReportGenerationService;
import com.iqser.red.service.redaction.report.v1.server.storage.ReportStorageService;
import com.iqser.red.service.redaction.v1.model.RedactionLog;
import com.iqser.red.storage.commons.service.StorageService;
import org.apache.commons.io.IOUtils;
import org.apache.poi.xwpf.usermodel.XWPFDocument;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.core.io.ClassPathResource;
import org.springframework.test.context.junit4.SpringRunner;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.Arrays;
import java.util.Base64;
import java.util.List;
import java.util.Map;
import static com.iqser.red.service.redaction.report.v1.server.utils.OsUtils.getTemporaryDirectory;
import static org.mockito.Mockito.when;
import static org.springframework.boot.test.context.SpringBootTest.WebEnvironment.RANDOM_PORT;
@RunWith(SpringRunner.class)
@SpringBootTest(webEnvironment = RANDOM_PORT)
public class RealDataServiceTest {
@Autowired
private ObjectMapper objectMapper;
@MockBean
private StorageService storageService;
@MockBean
private ReportStorageService reportStorageService;
@MockBean
private ReportTemplateClient reportTemplateClient;
@MockBean
private FileAttributesConfigClient fileAttributesConfigClient;
@MockBean
private AmazonS3 s3Client;
@Autowired
private WordReportGenerationService wordReportGenerationService;
@MockBean
private MessagingConfiguration messagingConfiguration;
@Autowired
private RedactionLogConverterService redactionLogConverterService;
@MockBean
private DossierAttributesConfigClient dossierAttributesConfigClient;
@MockBean
private DossierAttributesClient dossierAttributesClient;
@Autowired
private ExcelTemplateReportGenerationService excelTemplateReportGenerationService;
@Autowired
private GeneratePlaceholderService generatePlaceholderService;
@Test
public void testWordReportGeneration() throws IOException {
String dossierTemplateId = "dossierTemplateId";
ClassPathResource redactionLogResource = new ClassPathResource("realdata/redaction-log.json");
RedactionLog redactionLog = objectMapper.readValue(redactionLogResource.getInputStream(), RedactionLog.class);
ClassPathResource legalBasisMappingResource = new ClassPathResource("files/legalBasisMapping.json");
List<LegalBasis> legalBasisMapping = objectMapper.readValue(legalBasisMappingResource.getInputStream(), new TypeReference<>() {
});
List<ReportRedactionEntry> reportEntries = redactionLogConverterService.convertAndSort(redactionLog, legalBasisMapping);
FileModel fileStatus = FileModel.builder().filename("FileABCD").fileAttributes(Map.of("3e9b9569-5d2e-4619-848b-dd0a3e96527f", "Test")).build();
Dossier dossier = Dossier.builder().id("dossierId").dossierName("dossierName").build();
String templateId = "templateId";
String storageId = "storageId";
when(reportTemplateClient.getReportTemplate(dossierTemplateId, templateId)).thenReturn(ReportTemplate.builder()
.dossierTemplateId(dossierTemplateId)
.storageId(storageId)
.build());
ClassPathResource templateResource = new ClassPathResource("realdata/Justification Appendix A2.docx");
var placeholders = generatePlaceholderService.buildPlaceholders(dossier);
XWPFDocument doc = new XWPFDocument(templateResource.getInputStream());
wordReportGenerationService.generateReport(reportEntries, placeholders, "test", doc, fileStatus, dossier, true);
byte[] report = wordReportGenerationService.toByteArray(doc);
try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/efsa_template_wrg.docx")) {
fileOutputStream.write(report);
}
}
}