diff --git a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/CellTextChunkingService.java b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/CellTextChunkingService.java new file mode 100644 index 0000000..02d30a2 --- /dev/null +++ b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/CellTextChunkingService.java @@ -0,0 +1,33 @@ +package com.iqser.red.service.redaction.report.v1.server.service; + +import java.util.ArrayList; +import java.util.List; + +import org.springframework.stereotype.Service; + +@Service +public class CellTextChunkingService { + + public static final int MAX_CELL_TEXT_LENGTH = 32767; + + + public List process(String value) { + + List chunks = new ArrayList<>(); + if (value == null) { + return chunks; + } + + int length = value.length(); + int startIndex = 0; + + while (startIndex < length) { + int endIndex = Math.min(startIndex + MAX_CELL_TEXT_LENGTH, length); + String chunk = value.substring(startIndex, endIndex); + chunks.add(chunk); + startIndex = endIndex; + } + return chunks; + } + +} diff --git a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ComponentRowsReportService.java b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ComponentRowsReportService.java index 87eaf65..63be32f 100644 --- a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ComponentRowsReportService.java +++ b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ComponentRowsReportService.java @@ -1,18 +1,16 @@ package com.iqser.red.service.redaction.report.v1.server.service; +import java.util.*; import java.util.concurrent.atomic.AtomicInteger; -import org.apache.poi.ss.usermodel.Cell; -import org.apache.poi.ss.usermodel.Row; -import org.apache.poi.ss.usermodel.Sheet; +import org.apache.poi.ss.usermodel.*; import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog; -import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileAttributeConfig; -import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel; -import com.iqser.red.service.redaction.report.v1.server.client.ComponentClient; -import com.iqser.red.service.redaction.report.v1.server.client.FileAttributesConfigClient; -import com.iqser.red.service.redaction.report.v1.server.model.ExcelModel; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogEntryValue; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.*; +import com.iqser.red.service.redaction.report.v1.server.client.*; +import com.iqser.red.service.redaction.report.v1.server.model.*; import lombok.AccessLevel; import lombok.RequiredArgsConstructor; @@ -25,6 +23,7 @@ public class ComponentRowsReportService { ComponentClient componentResource; FileAttributesConfigClient fileAttributesClient; + CellTextChunkingService cellTextChunkingService; @SuppressWarnings("checkstyle:all") static int COMPONENT_NAME_COL = 0; @@ -39,30 +38,57 @@ public class ComponentRowsReportService { String oecd = getOecdNumber(fileModel); - componentLog.getComponentLogEntries().forEach(componentLogEntry -> { + componentLog.getComponentLogEntries() + .forEach(componentLogEntry -> { - if (componentLogEntry.getValues().isEmpty()) { - return; - } - int componentRowIdx = rowIndex.getAndIncrement(); - Row componentRow = sheet.createRow(componentRowIdx); - excelModel.getWrittenRows().add(componentRowIdx); + if (componentLogEntry.getValues().isEmpty()) { + return; + } - Cell componentNameCell = componentRow.createCell(COMPONENT_NAME_COL); - componentNameCell.setCellValue(oecd + "-" + componentLogEntry.getName().replaceAll("_", " ")); + String componentName = oecd + "-" + componentLogEntry.getName().replaceAll("_", " "); - for (int valueIdx = 0; valueIdx < componentLogEntry.getValues().size(); valueIdx++) { - String value = componentLogEntry.getValues().get(valueIdx).getValue(); - componentRow.createCell(COMPONENT_VALUE_STARTING_COL + valueIdx).setCellValue(value); - } + // Collect chunks for all values + List> valueChunksList = new ArrayList<>(); + int maxChunks = 0; - }); + for (ComponentLogEntryValue valueEntry : componentLogEntry.getValues()) { + String value = valueEntry.getValue(); + List chunks = cellTextChunkingService.process(value); + valueChunksList.add(chunks); + if (chunks.size() > maxChunks) { + maxChunks = chunks.size(); + } + } + + // Create rows for each chunk index + for (int chunkIdx = 0; chunkIdx < maxChunks; chunkIdx++) { + int componentRowIdx = rowIndex.getAndIncrement(); + Row componentRow = sheet.createRow(componentRowIdx); + excelModel.getWrittenRows().add(componentRowIdx); + + // Component Name Cell + Cell componentNameCell = componentRow.createCell(COMPONENT_NAME_COL); + componentNameCell.setCellValue(componentName); + + // Cells for each value + for (int valueIdx = 0; valueIdx < valueChunksList.size(); valueIdx++) { + List chunks = valueChunksList.get(valueIdx); + String chunk = chunkIdx < chunks.size() ? chunks.get(chunkIdx) : ""; + + Cell valueCell = componentRow.createCell(COMPONENT_VALUE_STARTING_COL + valueIdx); + valueCell.setCellValue(chunk); + CellStyle cellStyle = sheet.getWorkbook().createCellStyle(); + cellStyle.setWrapText(true); + valueCell.setCellStyle(cellStyle); + } + } + + }); sheet.createRow(rowIndex.get()); excelModel.getWrittenRows().add(rowIndex.get()); excelModel.setRedactionPlaceholderRow(rowIndex.getAndIncrement()); - } @@ -73,7 +99,8 @@ public class ComponentRowsReportService { .filter(f -> f.getLabel().equals("OECD Number")) .map(FileAttributeConfig::getId) .findFirst() - .map(oecd -> file.getFileAttributes().get(oecd)) + .map(oecd -> file.getFileAttributes() + .get(oecd)) .orElse(null); } diff --git a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ScmReportService.java b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ScmReportService.java index 0fc147f..ae10f11 100644 --- a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ScmReportService.java +++ b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ScmReportService.java @@ -6,13 +6,7 @@ import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; -import org.apache.poi.ss.usermodel.Cell; -import org.apache.poi.ss.usermodel.CellStyle; -import org.apache.poi.ss.usermodel.DataFormatter; -import org.apache.poi.ss.usermodel.HorizontalAlignment; -import org.apache.poi.ss.usermodel.Row; -import org.apache.poi.ss.usermodel.Sheet; -import org.apache.poi.ss.usermodel.VerticalAlignment; +import org.apache.poi.ss.usermodel.*; import org.apache.poi.ss.util.CellUtil; import org.apache.poi.xssf.streaming.SXSSFSheet; import org.springframework.stereotype.Service; @@ -36,10 +30,12 @@ import lombok.experimental.FieldDefaults; @SuppressWarnings("checkstyle:all") public class ScmReportService { + public static final int MAX_CELL_TEXT_LENGTH = 32767; final ComponentClient componentResource; final FileAttributesConfigClient fileAttributesClient; + final CellTextChunkingService cellTextChunkingService; final DataFormatter formatter = new DataFormatter(); - Row row; + public void addScmRows(Sheet sheet, FileModel fileModel, ExcelModel excelModel) { @@ -51,13 +47,10 @@ public class ScmReportService { firstRowIndex++; } - // sometimes first row is identified wrong. this makes sure we always identify the correct first row which contains the - // header cells as we need them for file attributes (e.g. OECD Number) + // Ensure we have the correct first row Cell firstCell = firstRow.getCell(0); - if (formatter.formatCellValue(firstCell).equals("File")) { - row = firstRow; - } else { - firstRow = row; + if (!formatter.formatCellValue(firstCell).equals("File")) { + firstRow = sheet.getRow(0); } int fileColumnIndex = 0; @@ -75,30 +68,32 @@ public class ScmReportService { Row finalFirstRow = firstRow; - componentLog.getComponentLogEntries().forEach(componentLogEntry -> { + componentLog.getComponentLogEntries() + .forEach(componentLogEntry -> { - if (componentLogEntry.getValues().isEmpty()) { - return; - } + if (componentLogEntry.getValues().isEmpty()) { + return; + } - int rowIndexAndIncrement = rowIndex.getAndIncrement(); - Row row = sheet.createRow(rowIndexAndIncrement); - excelModel.getWrittenRows().add(rowIndexAndIncrement); + // Process each ComponentLogEntryValue + addOtherCells(sheet, + excelModel, + rowIndex, + componentIndexMap, + componentLogEntry, + fileModel, + finalFirstRow, + fileColumnIndex, + componentColumnIndex, + valueIndexColumn, + textColumnIndex); - // Filename cell - Cell fileNameCell = row.createCell(fileColumnIndex); - fileNameCell.setCellValue(fileModel.getFilename() == null ? "" : fileModel.getFilename()); - CellUtil.setVerticalAlignment(fileNameCell, VerticalAlignment.TOP); - - // Component, file attributes and index cell - addOtherCells(sheet, excelModel, rowIndex, componentIndexMap, componentLogEntry, row, fileModel, finalFirstRow, componentColumnIndex, textColumnIndex, valueIndexColumn); - - }); + }); // Autosize all cells besides the last column because the extracted text should be multiline if (sheet instanceof SXSSFSheet) { ((SXSSFSheet) sheet).trackAllColumnsForAutoSizing(); - for (int i = 0; i < firstRow.getLastCellNum()-1; i++) { + for (int i = 0; i < firstRow.getLastCellNum() - 1; i++) { sheet.autoSizeColumn(i); } } @@ -114,77 +109,61 @@ public class ScmReportService { AtomicInteger rowIndex, Map componentIndexMap, ComponentLogEntry componentLogEntry, - Row row, FileModel fileModel, Row firstRow, + int fileColumnIndex, int componentColumnIndex, - int textColumnIndex, - int valueIndexColumn) { + int valueIndexColumn, + int textColumnIndex) { - int rowIndexAndIncrement; - - Cell componentNameCell = row.createCell(componentColumnIndex); String componentValue = componentLogEntry.getName().replaceAll("_", " "); - componentNameCell.setCellValue(componentValue); + + int componentIndex = componentIndexMap.getOrDefault(componentValue, 0); List fileAttributeModels = buildFileAttributeModels(fileModel, firstRow); - - // FileAttributes cells - addFileAttribute(row, fileModel, fileAttributeModels); - var iterator = componentLogEntry.getValues().iterator(); - while (iterator.hasNext()) { - ComponentLogEntryValue componentLogEntryValue = iterator.next(); - Cell indexCell = row.createCell(valueIndexColumn); - Cell textExtractionCell = row.createCell(textColumnIndex); + for (ComponentLogEntryValue componentLogEntryValue : componentLogEntry.getValues()) { - addTextExtraction(componentLogEntryValue, textExtractionCell); + componentIndex++; + componentIndexMap.put(componentValue, componentIndex); - if (componentIndexMap.containsKey(componentValue)) { - int indexValue = componentIndexMap.get(componentValue); - indexCell.setCellValue(indexValue + 1); + List textChunks = cellTextChunkingService.process(componentLogEntryValue.getValue()); - Cell componentCell = row.createCell(componentColumnIndex); - componentCell.setCellValue(componentValue); + for (String chunk : textChunks) { + int rowIndexAndIncrement = rowIndex.getAndIncrement(); + Row row = sheet.createRow(rowIndexAndIncrement); + excelModel.getWrittenRows().add(rowIndexAndIncrement); - Cell fileNameCell = row.createCell(0); - fileNameCell.setCellValue(fileModel.getFilename()); + Cell fileNameCell = row.createCell(fileColumnIndex); + fileNameCell.setCellValue(fileModel.getFilename() == null ? "" : fileModel.getFilename()); + CellUtil.setVerticalAlignment(fileNameCell, VerticalAlignment.TOP); + + Cell componentNameCell = row.createCell(componentColumnIndex); + componentNameCell.setCellValue(componentValue); + + Cell indexCell = row.createCell(valueIndexColumn); + indexCell.setCellValue(componentIndex); + CellUtil.setAlignment(indexCell, HorizontalAlignment.CENTER); + CellUtil.setVerticalAlignment(indexCell, VerticalAlignment.TOP); + + Cell textCell = row.createCell(textColumnIndex); + textCell.setCellValue(chunk); + CellStyle cellStyle = sheet.getWorkbook().createCellStyle(); + cellStyle.setWrapText(true); + textCell.setCellStyle(cellStyle); addFileAttribute(row, fileModel, fileAttributeModels); - - componentIndexMap.put(componentValue, indexValue + 1); - } else { - componentIndexMap.put(componentValue, 1); - indexCell.setCellValue(1); - } - - CellUtil.setAlignment(indexCell, HorizontalAlignment.CENTER); - CellUtil.setVerticalAlignment(componentNameCell, VerticalAlignment.TOP); - CellUtil.setVerticalAlignment(indexCell, VerticalAlignment.TOP); - - if (iterator.hasNext()) { - rowIndexAndIncrement = rowIndex.getAndIncrement(); - row = sheet.createRow(rowIndexAndIncrement); - excelModel.getWrittenRows().add(rowIndexAndIncrement); } } } - private void addTextExtraction(ComponentLogEntryValue componentLogEntryValue, Cell textExtractionCell) { - - textExtractionCell.setCellValue(componentLogEntryValue.getValue() == null ? "" : componentLogEntryValue.getValue()); - CellStyle cellStyle = textExtractionCell.getCellStyle(); - cellStyle.setWrapText(true); - textExtractionCell.setCellStyle(cellStyle); - } - - private void addFileAttribute(Row row, FileModel fileModel, List fileAttributeModels) { for (FileAttributeModel fileAttributeModel : fileAttributeModels) { Cell fileAttributeCell = row.createCell(fileAttributeModel.getCell().getColumnIndex()); - String cellValue = fileModel.getFileAttributes().get(fileAttributeModel.getId()); + String cellValue = fileModel.getFileAttributes() + .get(fileAttributeModel.getId()); fileAttributeCell.setCellValue(cellValue == null ? "" : cellValue); CellUtil.setAlignment(fileAttributeCell, HorizontalAlignment.CENTER); CellUtil.setVerticalAlignment(fileAttributeCell, VerticalAlignment.TOP); @@ -202,6 +181,7 @@ public class ScmReportService { return fileAttributes; } + private List buildFileAttributeModels(FileModel file, Row firstRow) { List fileAttributeModels = new ArrayList<>(); @@ -217,5 +197,4 @@ public class ScmReportService { return fileAttributeModels; } - } diff --git a/redaction-report-service-v1/redaction-report-service-server-v1/src/test/java/com/iqser/red/service/redaction/report/v1/server/RedactionReportIntegrationTest.java b/redaction-report-service-v1/redaction-report-service-server-v1/src/test/java/com/iqser/red/service/redaction/report/v1/server/RedactionReportIntegrationTest.java index da441ee..c9090f6 100644 --- a/redaction-report-service-v1/redaction-report-service-server-v1/src/test/java/com/iqser/red/service/redaction/report/v1/server/RedactionReportIntegrationTest.java +++ b/redaction-report-service-v1/redaction-report-service-server-v1/src/test/java/com/iqser/red/service/redaction/report/v1/server/RedactionReportIntegrationTest.java @@ -55,6 +55,8 @@ import org.springframework.test.context.junit.jupiter.SpringExtension; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogEntry; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogEntryValue; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogLegalBasis; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier; @@ -190,8 +192,9 @@ public class RedactionReportIntegrationTest { "{{dossier.attribute.Date}}", "2021-11-09T23:00:00.000Z"), Map.of("{{file.attribute.placeholder}}", "Test"), - List.of(new ImagePlaceholder("{{dossier.attribute.Signature}}", IOUtils.toByteArray(imageResource.getInputStream()), - ImagePlaceholder.ImageType.PNG))); + List.of(new ImagePlaceholder("{{dossier.attribute.Signature}}", + IOUtils.toByteArray(imageResource.getInputStream()), + ImagePlaceholder.ImageType.PNG))); XWPFDocument doc = new XWPFDocument(wordTemplateResource.getInputStream()); wordReportGenerationService.generateWordReport(reportEntries, placeholders, "test", doc, fileModel, dossier, true); @@ -241,7 +244,9 @@ public class RedactionReportIntegrationTest { "{{dossier.attribute.Date}}", "2021-11-09T23:00:00.000Z"), Map.of("{{file.attribute.placeholder}}", "Test"), - List.of(new ImagePlaceholder("{{dossier.attribute.Signature}}", IOUtils.toByteArray(imageResource.getInputStream()), ImagePlaceholder.ImageType.JPEG))); + List.of(new ImagePlaceholder("{{dossier.attribute.Signature}}", + IOUtils.toByteArray(imageResource.getInputStream()), + ImagePlaceholder.ImageType.JPEG))); XWPFDocument doc = new XWPFDocument(wordTemplateResource.getInputStream()); wordReportGenerationService.generateWordReport(reportEntries, placeholders, "test", doc, fileModel, dossier, true); @@ -405,7 +410,9 @@ public class RedactionReportIntegrationTest { "{{dossier.attribute.Date}}", "2021-11-09T23:00:00.000Z"), Map.of("{{file.attribute.placeholder}}", "Test"), - List.of(new ImagePlaceholder("{{dossier.attribute.Signature}}", IOUtils.toByteArray(imageResource.getInputStream()), ImagePlaceholder.ImageType.JPEG))); + List.of(new ImagePlaceholder("{{dossier.attribute.Signature}}", + IOUtils.toByteArray(imageResource.getInputStream()), + ImagePlaceholder.ImageType.JPEG))); XWPFDocument doc = new XWPFDocument(wordTemplateResource.getInputStream()); wordReportGenerationService.generateWordReport(reportEntries, placeholders, "test", doc, fileModel, dossier, true); @@ -546,7 +553,9 @@ public class RedactionReportIntegrationTest { var placeholders = buildPlaceHolderModel(Map.of("{{dossier_attribute.test1}}", "replaced_dossier_test1"), Map.of("{{file_attribute.test1}}", "replaced_file_test1", "{{file_attribute.test2}}", "replaced_file_test2"), - List.of(new ImagePlaceholder("{{dossier.attribute.Signature}}", IOUtils.toByteArray(imageResource.getInputStream()), ImagePlaceholder.ImageType.JPEG))); + List.of(new ImagePlaceholder("{{dossier.attribute.Signature}}", + IOUtils.toByteArray(imageResource.getInputStream()), + ImagePlaceholder.ImageType.JPEG))); XSSFWorkbook readWorkbook = new XSSFWorkbook(templateResource.getInputStream()); var excelModel = excelTemplateReportGenerationService.calculateExcelModel(readWorkbook.getSheetAt(0), dossier.getDossierTemplateId()); @@ -579,7 +588,9 @@ public class RedactionReportIntegrationTest { var placeholders = buildPlaceHolderModel(Map.of("{{dossier_attribute.test1}}", "replaced_dossier_test1"), Map.of("{{file_attribute.test1}}", "replaced_file_test1", "{{file_attribute.test2}}", "replaced_file_test2"), - List.of(new ImagePlaceholder("{{dossier.attribute.Signature}}", IOUtils.toByteArray(imageResource.getInputStream()), ImagePlaceholder.ImageType.JPEG))); + List.of(new ImagePlaceholder("{{dossier.attribute.Signature}}", + IOUtils.toByteArray(imageResource.getInputStream()), + ImagePlaceholder.ImageType.JPEG))); XSSFWorkbook readWorkbook = new XSSFWorkbook(templateResource.getInputStream()); var excelModel = excelTemplateReportGenerationService.calculateExcelModel(readWorkbook.getSheetAt(0), dossier.getDossierTemplateId()); @@ -636,6 +647,63 @@ public class RedactionReportIntegrationTest { } + @Test + @SneakyThrows + public void testComponentLogEntryExceedingMaxValue() { + + Dossier dossier = prepareDossier(); + + FileModel fileModel = FileModel.builder() + .filename("filename") + .dossierId(dossier.getId()) + .dossierTemplateId(dossier.getDossierTemplateId()) + .fileAttributes(Map.of("TestAttribute", "Lorem Ipsum")) + .build(); + + ComponentLogEntry componentLogEntry = createLargeComponentLogEntry(); + ComponentLog componentLog = new ComponentLog(); + componentLog.setComponentLogEntries(List.of(componentLogEntry)); + + when(componentClient.getComponentLog(dossier.getId(), fileModel.getId())).thenReturn(componentLog); + + List fileAttributeConfigs = objectMapper.readValue(new ClassPathResource("files/scm/fileAttributes.json").getInputStream(), new TypeReference<>() { + }); + when(fileAttributesConfigClient.getFileAttributeConfigs(dossier.getDossierTemplateId())).thenReturn(fileAttributeConfigs); + + ClassPathResource templateResource = new ClassPathResource("templates/scm_report.xlsx"); + XSSFWorkbook readWorkbook = new XSSFWorkbook(templateResource.getInputStream()); + var excelModel = excelTemplateReportGenerationService.calculateExcelModel(readWorkbook.getSheetAt(0), dossier.getDossierTemplateId()); + SXSSFWorkbook writeWorkbook = new SXSSFWorkbook(); + writeWorkbook.createSheet("Sheet1"); + + var placeholders = buildPlaceHolderModel(new HashMap<>(), new HashMap<>(), List.of()); + + excelTemplateReportGenerationService.generateExcelReport(new ArrayList<>(), placeholders, "test", writeWorkbook, dossier.getName(), fileModel, excelModel, true); + + + byte[] excelTemplateReport3 = excelTemplateReportGenerationService.toByteArray(writeWorkbook); + try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/scm_report.xlsx")) { + fileOutputStream.write(excelTemplateReport3); + } + System.out.println(getTemporaryDirectory() + "/scm_report.xlsx"); + } + + + private static ComponentLogEntry createLargeComponentLogEntry() { + + int length = 35000; // Exceed the max cell length of 32,767 + String longValue = "A".repeat(length); + + ComponentLogEntryValue componentLogEntryValue = new ComponentLogEntryValue(); + componentLogEntryValue.setValue(longValue); + + ComponentLogEntry componentLogEntry = new ComponentLogEntry(); + componentLogEntry.setName("TestComponent"); + componentLogEntry.setValues(List.of(componentLogEntryValue)); + return componentLogEntry; + } + + private Map createEntityDisplayNames(EntityLog entityLog) { Type t1 = new Type();