RED-10435: RM-216: Report generation fails when component is to long

This commit is contained in:
maverickstuder 2024-11-11 11:59:43 +01:00
parent e51fc5eb74
commit bb0bbe7fb6
4 changed files with 217 additions and 110 deletions

View File

@ -0,0 +1,33 @@
package com.iqser.red.service.redaction.report.v1.server.service;
import java.util.ArrayList;
import java.util.List;
import org.springframework.stereotype.Service;
@Service
public class CellTextChunkingService {
public static final int MAX_CELL_TEXT_LENGTH = 32767;
public List<String> process(String value) {
List<String> chunks = new ArrayList<>();
if (value == null) {
return chunks;
}
int length = value.length();
int startIndex = 0;
while (startIndex < length) {
int endIndex = Math.min(startIndex + MAX_CELL_TEXT_LENGTH, length);
String chunk = value.substring(startIndex, endIndex);
chunks.add(chunk);
startIndex = endIndex;
}
return chunks;
}
}

View File

@ -1,18 +1,16 @@
package com.iqser.red.service.redaction.report.v1.server.service;
import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.*;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileAttributeConfig;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
import com.iqser.red.service.redaction.report.v1.server.client.ComponentClient;
import com.iqser.red.service.redaction.report.v1.server.client.FileAttributesConfigClient;
import com.iqser.red.service.redaction.report.v1.server.model.ExcelModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogEntryValue;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.*;
import com.iqser.red.service.redaction.report.v1.server.client.*;
import com.iqser.red.service.redaction.report.v1.server.model.*;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
@ -25,6 +23,7 @@ public class ComponentRowsReportService {
ComponentClient componentResource;
FileAttributesConfigClient fileAttributesClient;
CellTextChunkingService cellTextChunkingService;
@SuppressWarnings("checkstyle:all")
static int COMPONENT_NAME_COL = 0;
@ -39,30 +38,57 @@ public class ComponentRowsReportService {
String oecd = getOecdNumber(fileModel);
componentLog.getComponentLogEntries().forEach(componentLogEntry -> {
componentLog.getComponentLogEntries()
.forEach(componentLogEntry -> {
if (componentLogEntry.getValues().isEmpty()) {
return;
}
int componentRowIdx = rowIndex.getAndIncrement();
Row componentRow = sheet.createRow(componentRowIdx);
excelModel.getWrittenRows().add(componentRowIdx);
if (componentLogEntry.getValues().isEmpty()) {
return;
}
Cell componentNameCell = componentRow.createCell(COMPONENT_NAME_COL);
componentNameCell.setCellValue(oecd + "-" + componentLogEntry.getName().replaceAll("_", " "));
String componentName = oecd + "-" + componentLogEntry.getName().replaceAll("_", " ");
for (int valueIdx = 0; valueIdx < componentLogEntry.getValues().size(); valueIdx++) {
String value = componentLogEntry.getValues().get(valueIdx).getValue();
componentRow.createCell(COMPONENT_VALUE_STARTING_COL + valueIdx).setCellValue(value);
}
// Collect chunks for all values
List<List<String>> valueChunksList = new ArrayList<>();
int maxChunks = 0;
});
for (ComponentLogEntryValue valueEntry : componentLogEntry.getValues()) {
String value = valueEntry.getValue();
List<String> chunks = cellTextChunkingService.process(value);
valueChunksList.add(chunks);
if (chunks.size() > maxChunks) {
maxChunks = chunks.size();
}
}
// Create rows for each chunk index
for (int chunkIdx = 0; chunkIdx < maxChunks; chunkIdx++) {
int componentRowIdx = rowIndex.getAndIncrement();
Row componentRow = sheet.createRow(componentRowIdx);
excelModel.getWrittenRows().add(componentRowIdx);
// Component Name Cell
Cell componentNameCell = componentRow.createCell(COMPONENT_NAME_COL);
componentNameCell.setCellValue(componentName);
// Cells for each value
for (int valueIdx = 0; valueIdx < valueChunksList.size(); valueIdx++) {
List<String> chunks = valueChunksList.get(valueIdx);
String chunk = chunkIdx < chunks.size() ? chunks.get(chunkIdx) : "";
Cell valueCell = componentRow.createCell(COMPONENT_VALUE_STARTING_COL + valueIdx);
valueCell.setCellValue(chunk);
CellStyle cellStyle = sheet.getWorkbook().createCellStyle();
cellStyle.setWrapText(true);
valueCell.setCellStyle(cellStyle);
}
}
});
sheet.createRow(rowIndex.get());
excelModel.getWrittenRows().add(rowIndex.get());
excelModel.setRedactionPlaceholderRow(rowIndex.getAndIncrement());
}
@ -73,7 +99,8 @@ public class ComponentRowsReportService {
.filter(f -> f.getLabel().equals("OECD Number"))
.map(FileAttributeConfig::getId)
.findFirst()
.map(oecd -> file.getFileAttributes().get(oecd))
.map(oecd -> file.getFileAttributes()
.get(oecd))
.orElse(null);
}

View File

@ -6,13 +6,7 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.CellStyle;
import org.apache.poi.ss.usermodel.DataFormatter;
import org.apache.poi.ss.usermodel.HorizontalAlignment;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.VerticalAlignment;
import org.apache.poi.ss.usermodel.*;
import org.apache.poi.ss.util.CellUtil;
import org.apache.poi.xssf.streaming.SXSSFSheet;
import org.springframework.stereotype.Service;
@ -36,10 +30,12 @@ import lombok.experimental.FieldDefaults;
@SuppressWarnings("checkstyle:all")
public class ScmReportService {
public static final int MAX_CELL_TEXT_LENGTH = 32767;
final ComponentClient componentResource;
final FileAttributesConfigClient fileAttributesClient;
final CellTextChunkingService cellTextChunkingService;
final DataFormatter formatter = new DataFormatter();
Row row;
public void addScmRows(Sheet sheet, FileModel fileModel, ExcelModel excelModel) {
@ -51,13 +47,10 @@ public class ScmReportService {
firstRowIndex++;
}
// sometimes first row is identified wrong. this makes sure we always identify the correct first row which contains the
// header cells as we need them for file attributes (e.g. OECD Number)
// Ensure we have the correct first row
Cell firstCell = firstRow.getCell(0);
if (formatter.formatCellValue(firstCell).equals("File")) {
row = firstRow;
} else {
firstRow = row;
if (!formatter.formatCellValue(firstCell).equals("File")) {
firstRow = sheet.getRow(0);
}
int fileColumnIndex = 0;
@ -75,30 +68,32 @@ public class ScmReportService {
Row finalFirstRow = firstRow;
componentLog.getComponentLogEntries().forEach(componentLogEntry -> {
componentLog.getComponentLogEntries()
.forEach(componentLogEntry -> {
if (componentLogEntry.getValues().isEmpty()) {
return;
}
if (componentLogEntry.getValues().isEmpty()) {
return;
}
int rowIndexAndIncrement = rowIndex.getAndIncrement();
Row row = sheet.createRow(rowIndexAndIncrement);
excelModel.getWrittenRows().add(rowIndexAndIncrement);
// Process each ComponentLogEntryValue
addOtherCells(sheet,
excelModel,
rowIndex,
componentIndexMap,
componentLogEntry,
fileModel,
finalFirstRow,
fileColumnIndex,
componentColumnIndex,
valueIndexColumn,
textColumnIndex);
// Filename cell
Cell fileNameCell = row.createCell(fileColumnIndex);
fileNameCell.setCellValue(fileModel.getFilename() == null ? "" : fileModel.getFilename());
CellUtil.setVerticalAlignment(fileNameCell, VerticalAlignment.TOP);
// Component, file attributes and index cell
addOtherCells(sheet, excelModel, rowIndex, componentIndexMap, componentLogEntry, row, fileModel, finalFirstRow, componentColumnIndex, textColumnIndex, valueIndexColumn);
});
});
// Autosize all cells besides the last column because the extracted text should be multiline
if (sheet instanceof SXSSFSheet) {
((SXSSFSheet) sheet).trackAllColumnsForAutoSizing();
for (int i = 0; i < firstRow.getLastCellNum()-1; i++) {
for (int i = 0; i < firstRow.getLastCellNum() - 1; i++) {
sheet.autoSizeColumn(i);
}
}
@ -114,77 +109,61 @@ public class ScmReportService {
AtomicInteger rowIndex,
Map<String, Integer> componentIndexMap,
ComponentLogEntry componentLogEntry,
Row row,
FileModel fileModel,
Row firstRow,
int fileColumnIndex,
int componentColumnIndex,
int textColumnIndex,
int valueIndexColumn) {
int valueIndexColumn,
int textColumnIndex) {
int rowIndexAndIncrement;
Cell componentNameCell = row.createCell(componentColumnIndex);
String componentValue = componentLogEntry.getName().replaceAll("_", " ");
componentNameCell.setCellValue(componentValue);
int componentIndex = componentIndexMap.getOrDefault(componentValue, 0);
List<FileAttributeModel> fileAttributeModels = buildFileAttributeModels(fileModel, firstRow);
// FileAttributes cells
addFileAttribute(row, fileModel, fileAttributeModels);
var iterator = componentLogEntry.getValues().iterator();
while (iterator.hasNext()) {
ComponentLogEntryValue componentLogEntryValue = iterator.next();
Cell indexCell = row.createCell(valueIndexColumn);
Cell textExtractionCell = row.createCell(textColumnIndex);
for (ComponentLogEntryValue componentLogEntryValue : componentLogEntry.getValues()) {
addTextExtraction(componentLogEntryValue, textExtractionCell);
componentIndex++;
componentIndexMap.put(componentValue, componentIndex);
if (componentIndexMap.containsKey(componentValue)) {
int indexValue = componentIndexMap.get(componentValue);
indexCell.setCellValue(indexValue + 1);
List<String> textChunks = cellTextChunkingService.process(componentLogEntryValue.getValue());
Cell componentCell = row.createCell(componentColumnIndex);
componentCell.setCellValue(componentValue);
for (String chunk : textChunks) {
int rowIndexAndIncrement = rowIndex.getAndIncrement();
Row row = sheet.createRow(rowIndexAndIncrement);
excelModel.getWrittenRows().add(rowIndexAndIncrement);
Cell fileNameCell = row.createCell(0);
fileNameCell.setCellValue(fileModel.getFilename());
Cell fileNameCell = row.createCell(fileColumnIndex);
fileNameCell.setCellValue(fileModel.getFilename() == null ? "" : fileModel.getFilename());
CellUtil.setVerticalAlignment(fileNameCell, VerticalAlignment.TOP);
Cell componentNameCell = row.createCell(componentColumnIndex);
componentNameCell.setCellValue(componentValue);
Cell indexCell = row.createCell(valueIndexColumn);
indexCell.setCellValue(componentIndex);
CellUtil.setAlignment(indexCell, HorizontalAlignment.CENTER);
CellUtil.setVerticalAlignment(indexCell, VerticalAlignment.TOP);
Cell textCell = row.createCell(textColumnIndex);
textCell.setCellValue(chunk);
CellStyle cellStyle = sheet.getWorkbook().createCellStyle();
cellStyle.setWrapText(true);
textCell.setCellStyle(cellStyle);
addFileAttribute(row, fileModel, fileAttributeModels);
componentIndexMap.put(componentValue, indexValue + 1);
} else {
componentIndexMap.put(componentValue, 1);
indexCell.setCellValue(1);
}
CellUtil.setAlignment(indexCell, HorizontalAlignment.CENTER);
CellUtil.setVerticalAlignment(componentNameCell, VerticalAlignment.TOP);
CellUtil.setVerticalAlignment(indexCell, VerticalAlignment.TOP);
if (iterator.hasNext()) {
rowIndexAndIncrement = rowIndex.getAndIncrement();
row = sheet.createRow(rowIndexAndIncrement);
excelModel.getWrittenRows().add(rowIndexAndIncrement);
}
}
}
private void addTextExtraction(ComponentLogEntryValue componentLogEntryValue, Cell textExtractionCell) {
textExtractionCell.setCellValue(componentLogEntryValue.getValue() == null ? "" : componentLogEntryValue.getValue());
CellStyle cellStyle = textExtractionCell.getCellStyle();
cellStyle.setWrapText(true);
textExtractionCell.setCellStyle(cellStyle);
}
private void addFileAttribute(Row row, FileModel fileModel, List<FileAttributeModel> fileAttributeModels) {
for (FileAttributeModel fileAttributeModel : fileAttributeModels) {
Cell fileAttributeCell = row.createCell(fileAttributeModel.getCell().getColumnIndex());
String cellValue = fileModel.getFileAttributes().get(fileAttributeModel.getId());
String cellValue = fileModel.getFileAttributes()
.get(fileAttributeModel.getId());
fileAttributeCell.setCellValue(cellValue == null ? "" : cellValue);
CellUtil.setAlignment(fileAttributeCell, HorizontalAlignment.CENTER);
CellUtil.setVerticalAlignment(fileAttributeCell, VerticalAlignment.TOP);
@ -202,6 +181,7 @@ public class ScmReportService {
return fileAttributes;
}
private List<FileAttributeModel> buildFileAttributeModels(FileModel file, Row firstRow) {
List<FileAttributeModel> fileAttributeModels = new ArrayList<>();
@ -217,5 +197,4 @@ public class ScmReportService {
return fileAttributeModels;
}
}

View File

@ -55,6 +55,8 @@ import org.springframework.test.context.junit.jupiter.SpringExtension;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogEntryValue;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogLegalBasis;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier;
@ -190,8 +192,9 @@ public class RedactionReportIntegrationTest {
"{{dossier.attribute.Date}}",
"2021-11-09T23:00:00.000Z"),
Map.of("{{file.attribute.placeholder}}", "Test"),
List.of(new ImagePlaceholder("{{dossier.attribute.Signature}}", IOUtils.toByteArray(imageResource.getInputStream()),
ImagePlaceholder.ImageType.PNG)));
List.of(new ImagePlaceholder("{{dossier.attribute.Signature}}",
IOUtils.toByteArray(imageResource.getInputStream()),
ImagePlaceholder.ImageType.PNG)));
XWPFDocument doc = new XWPFDocument(wordTemplateResource.getInputStream());
wordReportGenerationService.generateWordReport(reportEntries, placeholders, "test", doc, fileModel, dossier, true);
@ -241,7 +244,9 @@ public class RedactionReportIntegrationTest {
"{{dossier.attribute.Date}}",
"2021-11-09T23:00:00.000Z"),
Map.of("{{file.attribute.placeholder}}", "Test"),
List.of(new ImagePlaceholder("{{dossier.attribute.Signature}}", IOUtils.toByteArray(imageResource.getInputStream()), ImagePlaceholder.ImageType.JPEG)));
List.of(new ImagePlaceholder("{{dossier.attribute.Signature}}",
IOUtils.toByteArray(imageResource.getInputStream()),
ImagePlaceholder.ImageType.JPEG)));
XWPFDocument doc = new XWPFDocument(wordTemplateResource.getInputStream());
wordReportGenerationService.generateWordReport(reportEntries, placeholders, "test", doc, fileModel, dossier, true);
@ -405,7 +410,9 @@ public class RedactionReportIntegrationTest {
"{{dossier.attribute.Date}}",
"2021-11-09T23:00:00.000Z"),
Map.of("{{file.attribute.placeholder}}", "Test"),
List.of(new ImagePlaceholder("{{dossier.attribute.Signature}}", IOUtils.toByteArray(imageResource.getInputStream()), ImagePlaceholder.ImageType.JPEG)));
List.of(new ImagePlaceholder("{{dossier.attribute.Signature}}",
IOUtils.toByteArray(imageResource.getInputStream()),
ImagePlaceholder.ImageType.JPEG)));
XWPFDocument doc = new XWPFDocument(wordTemplateResource.getInputStream());
wordReportGenerationService.generateWordReport(reportEntries, placeholders, "test", doc, fileModel, dossier, true);
@ -546,7 +553,9 @@ public class RedactionReportIntegrationTest {
var placeholders = buildPlaceHolderModel(Map.of("{{dossier_attribute.test1}}", "replaced_dossier_test1"),
Map.of("{{file_attribute.test1}}", "replaced_file_test1", "{{file_attribute.test2}}", "replaced_file_test2"),
List.of(new ImagePlaceholder("{{dossier.attribute.Signature}}", IOUtils.toByteArray(imageResource.getInputStream()), ImagePlaceholder.ImageType.JPEG)));
List.of(new ImagePlaceholder("{{dossier.attribute.Signature}}",
IOUtils.toByteArray(imageResource.getInputStream()),
ImagePlaceholder.ImageType.JPEG)));
XSSFWorkbook readWorkbook = new XSSFWorkbook(templateResource.getInputStream());
var excelModel = excelTemplateReportGenerationService.calculateExcelModel(readWorkbook.getSheetAt(0), dossier.getDossierTemplateId());
@ -579,7 +588,9 @@ public class RedactionReportIntegrationTest {
var placeholders = buildPlaceHolderModel(Map.of("{{dossier_attribute.test1}}", "replaced_dossier_test1"),
Map.of("{{file_attribute.test1}}", "replaced_file_test1", "{{file_attribute.test2}}", "replaced_file_test2"),
List.of(new ImagePlaceholder("{{dossier.attribute.Signature}}", IOUtils.toByteArray(imageResource.getInputStream()), ImagePlaceholder.ImageType.JPEG)));
List.of(new ImagePlaceholder("{{dossier.attribute.Signature}}",
IOUtils.toByteArray(imageResource.getInputStream()),
ImagePlaceholder.ImageType.JPEG)));
XSSFWorkbook readWorkbook = new XSSFWorkbook(templateResource.getInputStream());
var excelModel = excelTemplateReportGenerationService.calculateExcelModel(readWorkbook.getSheetAt(0), dossier.getDossierTemplateId());
@ -636,6 +647,63 @@ public class RedactionReportIntegrationTest {
}
@Test
@SneakyThrows
public void testComponentLogEntryExceedingMaxValue() {
Dossier dossier = prepareDossier();
FileModel fileModel = FileModel.builder()
.filename("filename")
.dossierId(dossier.getId())
.dossierTemplateId(dossier.getDossierTemplateId())
.fileAttributes(Map.of("TestAttribute", "Lorem Ipsum"))
.build();
ComponentLogEntry componentLogEntry = createLargeComponentLogEntry();
ComponentLog componentLog = new ComponentLog();
componentLog.setComponentLogEntries(List.of(componentLogEntry));
when(componentClient.getComponentLog(dossier.getId(), fileModel.getId())).thenReturn(componentLog);
List<FileAttributeConfig> fileAttributeConfigs = objectMapper.readValue(new ClassPathResource("files/scm/fileAttributes.json").getInputStream(), new TypeReference<>() {
});
when(fileAttributesConfigClient.getFileAttributeConfigs(dossier.getDossierTemplateId())).thenReturn(fileAttributeConfigs);
ClassPathResource templateResource = new ClassPathResource("templates/scm_report.xlsx");
XSSFWorkbook readWorkbook = new XSSFWorkbook(templateResource.getInputStream());
var excelModel = excelTemplateReportGenerationService.calculateExcelModel(readWorkbook.getSheetAt(0), dossier.getDossierTemplateId());
SXSSFWorkbook writeWorkbook = new SXSSFWorkbook();
writeWorkbook.createSheet("Sheet1");
var placeholders = buildPlaceHolderModel(new HashMap<>(), new HashMap<>(), List.of());
excelTemplateReportGenerationService.generateExcelReport(new ArrayList<>(), placeholders, "test", writeWorkbook, dossier.getName(), fileModel, excelModel, true);
byte[] excelTemplateReport3 = excelTemplateReportGenerationService.toByteArray(writeWorkbook);
try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/scm_report.xlsx")) {
fileOutputStream.write(excelTemplateReport3);
}
System.out.println(getTemporaryDirectory() + "/scm_report.xlsx");
}
private static ComponentLogEntry createLargeComponentLogEntry() {
int length = 35000; // Exceed the max cell length of 32,767
String longValue = "A".repeat(length);
ComponentLogEntryValue componentLogEntryValue = new ComponentLogEntryValue();
componentLogEntryValue.setValue(longValue);
ComponentLogEntry componentLogEntry = new ComponentLogEntry();
componentLogEntry.setName("TestComponent");
componentLogEntry.setValues(List.of(componentLogEntryValue));
return componentLogEntry;
}
private Map<String, String> createEntityDisplayNames(EntityLog entityLog) {
Type t1 = new Type();