From 808814911c3ce25f69df80f3c96e5ac6f2e4b412 Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Mon, 16 Dec 2024 16:21:53 +0100 Subject: [PATCH] RED-10683: fix NPE for multi file reports --- .../v1/server/model/ComponentReportModel.java | 50 +++ .../report/v1/server/model/ExcelModel.java | 9 +- .../v1/server/service/ExcelModelFactory.java | 268 ++++++++++++++ .../service/ExcelReportGenerationService.java | 341 ++++-------------- .../service/ExcelReportTemplateService.java | 17 +- .../service/GeneratePlaceholderService.java | 4 +- .../v1/server/service/PlaceholderService.java | 2 +- .../service/ReportGenerationService.java | 19 +- .../v1/server/service/ScmReportService.java | 105 ++---- .../RedactionReportIntegrationTest.java | 18 +- .../src/test/resources/logback-spring.xml | 17 + 11 files changed, 484 insertions(+), 366 deletions(-) create mode 100644 redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/model/ComponentReportModel.java create mode 100644 redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ExcelModelFactory.java create mode 100644 redaction-report-service-v1/redaction-report-service-server-v1/src/test/resources/logback-spring.xml diff --git a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/model/ComponentReportModel.java b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/model/ComponentReportModel.java new file mode 100644 index 0000000..e92ab79 --- /dev/null +++ b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/model/ComponentReportModel.java @@ -0,0 +1,50 @@ +package com.iqser.red.service.redaction.report.v1.server.model; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.experimental.FieldDefaults; + +@AllArgsConstructor +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) +public class ComponentReportModel { + + Integer componentStartColumn; + Integer componentValueIndexColumn; + + + public boolean isComponentReport() { + + return componentStartColumn != null; + } + + + public int getComponentNameColumn() { + + assert isComponentReport(); + return componentStartColumn; + } + + + public int getComponentValueIndexColumn() { + + assert writeComponentValueIndices(); + return componentValueIndexColumn; + } + + + public int getComponentValueColumn() { + + assert isComponentReport(); + if (writeComponentValueIndices()) { + return componentValueIndexColumn + 1; + } + return componentStartColumn + 1; + } + + + public boolean writeComponentValueIndices() { + + return componentValueIndexColumn != null; + } + +} diff --git a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/model/ExcelModel.java b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/model/ExcelModel.java index e7b7f72..d42b786 100644 --- a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/model/ExcelModel.java +++ b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/model/ExcelModel.java @@ -22,14 +22,19 @@ public class ExcelModel { private Map cellWidths = new HashMap<>(); private Map cellsToCopyBeforeRedactionPlaceholderRow = new HashMap<>(); private Map cellsToCopyAfterRedactionPlaceholderRow = new HashMap<>(); - private List writtenRows = new ArrayList<>(); private boolean rowsBeforeRedactionEntryRowsAdded; private boolean rssPlaceholdersPresent; private boolean placeholderInFirstRow; private boolean firstRowWritten; private boolean skippedPlaceholderPresent; - private boolean fileAttributesPlaceholderPresent; private boolean scmFunctionPlaceholderPresent; + private ComponentReportModel componentReport; + private List fileAttributeColumns; + + public boolean isFileAttributesPlaceholderPresent() { + + return !fileAttributeColumns.isEmpty(); + } } diff --git a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ExcelModelFactory.java b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ExcelModelFactory.java new file mode 100644 index 0000000..a1ba01b --- /dev/null +++ b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ExcelModelFactory.java @@ -0,0 +1,268 @@ +package com.iqser.red.service.redaction.report.v1.server.service; + +import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.COMPONENT_PLACEHOLDER; +import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.COMPONENT_PLACEHOLDER_BASE; +import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.DOSSIER_ATTRIBUTE_PLACEHOLDER_BASE; +import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.EXCERPT_PLACEHOLDER; +import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.FILE_ATTRIBUTES_PLACEHOLDER; +import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.FILE_ATTRIBUTE_PLACEHOLDER_BASE; +import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.FILE_NAME_PLACEHOLDER; +import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.INDEX_PLACEHOLDER; +import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.JUSTIFICATION_LEGAL_BASIS_PLACEHOLDER; +import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.JUSTIFICATION_PARAGRAPH_PLACEHOLDER; +import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.JUSTIFICATION_PLACEHOLDER; +import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.JUSTIFICATION_REASON_PLACEHOLDER; +import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.JUSTIFICATION_TEXT_PLACEHOLDER; +import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.PAGE_PLACEHOLDER; +import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.PARAGRAPH_INDEX_PLACEHOLDER; +import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.PARAGRAPH_PLACEHOLDER; +import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.REDACTION_ENTITY_DISPLAY_NAME_PLACEHOLDER; +import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.REDACTION_VALUE_PLACEHOLDER; +import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.SCM_FUNCTION_PLACEHOLDER; +import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.SEEDS_FUNCTION_JUSTIFICATION_PLACEHOLDER; +import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.SEEDS_FUNCTION_REDACTION_GROUPED_BY_JUSTIFICATION_PAGES_PLACEHOLDER; +import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.SKIPPED_PLACEHOLDER; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; + +import org.apache.poi.ss.usermodel.Cell; +import org.apache.poi.ss.usermodel.CellStyle; +import org.apache.poi.ss.usermodel.HorizontalAlignment; +import org.apache.poi.ss.usermodel.Row; +import org.apache.poi.ss.usermodel.Sheet; +import org.springframework.stereotype.Service; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileAttributeConfig; +import com.iqser.red.service.redaction.report.v1.server.client.FileAttributesConfigClient; +import com.iqser.red.service.redaction.report.v1.server.model.CellIdentifier; +import com.iqser.red.service.redaction.report.v1.server.model.ComponentReportModel; +import com.iqser.red.service.redaction.report.v1.server.model.ExcelModel; +import com.iqser.red.service.redaction.report.v1.server.model.FileAttributeModel; +import com.iqser.red.service.redaction.report.v1.server.model.PlaceholderInput; + +import io.micrometer.core.annotation.Timed; +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; +import lombok.experimental.FieldDefaults; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +@RequiredArgsConstructor +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) +public class ExcelModelFactory { + + private static final Set prefixRedactionPlaceholders = Set.of(DOSSIER_ATTRIBUTE_PLACEHOLDER_BASE, FILE_ATTRIBUTE_PLACEHOLDER_BASE); + private static final Set redactionPlaceholders = Set.of(FILE_NAME_PLACEHOLDER, + PAGE_PLACEHOLDER, + PARAGRAPH_PLACEHOLDER, + PARAGRAPH_INDEX_PLACEHOLDER, + JUSTIFICATION_PLACEHOLDER, + EXCERPT_PLACEHOLDER, + JUSTIFICATION_PARAGRAPH_PLACEHOLDER, + JUSTIFICATION_REASON_PLACEHOLDER, + REDACTION_VALUE_PLACEHOLDER, + JUSTIFICATION_LEGAL_BASIS_PLACEHOLDER, + JUSTIFICATION_TEXT_PLACEHOLDER, + SEEDS_FUNCTION_REDACTION_GROUPED_BY_JUSTIFICATION_PAGES_PLACEHOLDER, + SEEDS_FUNCTION_JUSTIFICATION_PLACEHOLDER, + REDACTION_ENTITY_DISPLAY_NAME_PLACEHOLDER, + SCM_FUNCTION_PLACEHOLDER, + SKIPPED_PLACEHOLDER, + FILE_ATTRIBUTES_PLACEHOLDER, + INDEX_PLACEHOLDER, + COMPONENT_PLACEHOLDER); + + FileAttributesConfigClient fileAttributesConfigClient; + + + @Timed("redactmanager_calculateExcelModel") + @SuppressWarnings("checkstyle:all") + public ExcelModel calculateExcelModel(Sheet sheet, String dossierTemplateId) { + + long start = System.currentTimeMillis(); + + Map> placeholderCellPos = new HashMap<>(); + Map columnWidths = new HashMap<>(); + Map cellsToCopyBeforePlaceholderRow = new HashMap<>(); + Map cellsToCopyAfterPlaceholderRow = new HashMap<>(); + Map cellsToCopyInPlaceholderRow = new HashMap<>(); + Integer componentColumn = null; + Integer componentValueIndexColumn = null; + List fileAttributeCols = new ArrayList<>(); + + boolean hasRssPlaceHolders = false; + boolean hasSkippedPlaceholder = false; + int placeholderRow = -1; + boolean placeholderInFirstRow = false; + boolean hasScmFunctionPlaceholder = false; + + for (int row = 0; row < sheet.getLastRowNum() + 1; row++) { + Row actualRow = sheet.getRow(row); + + if (actualRow == null) { + continue; + } + + int lastCellIndex = actualRow.getLastCellNum(); + for (int col = 0; col < lastCellIndex; col++) { + + Cell cell = actualRow.getCell(col); + + if (cell == null) { + continue; + } + + boolean isFileAttributesPlaceholder = false; + int columnWidth = sheet.getColumnWidth(col); + columnWidths.put(col, columnWidth); + String cellStringValue = cell.getStringCellValue(); + + if (row == 0 && cellStringValue.contains("{{")) { + placeholderInFirstRow = true; + } + + if (cellStringValue.contains(COMPONENT_PLACEHOLDER_BASE)) { + hasRssPlaceHolders = true; + } + + if (cellStringValue.contains(COMPONENT_PLACEHOLDER)) { + componentColumn = col; + } + + if (cellStringValue.contains(INDEX_PLACEHOLDER)) { + componentValueIndexColumn = col; + } + + if (cellStringValue.contains(SCM_FUNCTION_PLACEHOLDER)) { + hasScmFunctionPlaceholder = true; + } + + if (cellStringValue.contains(FILE_ATTRIBUTES_PLACEHOLDER)) { + + // For each file attribute we have to replace the column header with the corresponding file attribute. + List fileAttributeConfigs = fileAttributesConfigClient.getFileAttributeConfigs(dossierTemplateId); + + var iterator = fileAttributeConfigs.iterator(); + + while (iterator.hasNext()) { + + cell = getOrCreateCell(actualRow, col); + FileAttributeConfig fileAttributeConfig = iterator.next(); + cell.setCellValue(fileAttributeConfig.getLabel()); + cellsToCopyBeforePlaceholderRow.put(new CellIdentifier(row, col), cell); + fileAttributeCols.add(FileAttributeModel.builder().id(fileAttributeConfig.getId()).cell(cell).build()); + // If there is more than one file attribute we have to insert a new column, so we shift all columns to the right by 1 + // and increase the current and last cell index. + if (iterator.hasNext()) { + lastCellIndex += 1; + col += 1; + sheet.shiftColumns(col, lastCellIndex, 1); + } + } + + isFileAttributesPlaceholder = true; + } + + if (!isFileAttributesPlaceholder) { + if (isRedactionPlaceholder(cellStringValue)) { + if (cellStringValue.equals(SKIPPED_PLACEHOLDER)) { + hasSkippedPlaceholder = true; + } + placeholderCellPos.put(col, getFunctionForPlaceHolder(cellStringValue)); + placeholderRow = row; + cellsToCopyInPlaceholderRow.put(new CellIdentifier(row, col), cell); + } else { + if (placeholderRow == -1) { + cellsToCopyBeforePlaceholderRow.put(new CellIdentifier(row, col), cell); + } else { + cellsToCopyAfterPlaceholderRow.put(new CellIdentifier(row, col), cell); + } + } + } + } + } + + if (hasRssPlaceHolders) { + cellsToCopyBeforePlaceholderRow.putAll(cellsToCopyInPlaceholderRow); + cellsToCopyBeforePlaceholderRow.putAll(cellsToCopyAfterPlaceholderRow); + cellsToCopyAfterPlaceholderRow = new HashMap<>(); + } + + log.debug("Calculate Placeholder Cells took: {}", System.currentTimeMillis() - start); + ComponentReportModel componentReport = new ComponentReportModel(componentColumn, componentValueIndexColumn); + return new ExcelModel(placeholderCellPos, + placeholderRow, + columnWidths, + cellsToCopyBeforePlaceholderRow, + cellsToCopyAfterPlaceholderRow, + new ArrayList<>(), + false, + hasRssPlaceHolders, + placeholderInFirstRow, + false, + hasSkippedPlaceholder, + hasScmFunctionPlaceholder, + componentReport, + fileAttributeCols); + } + + + private static Cell getOrCreateCell(Row actualRow, int i) { + + Cell cell; + cell = actualRow.getCell(i); + + // If there is more than one file attribute, starting from the 2nd one we won't find a cell in the given index so we have to create it. + if (cell == null) { + cell = actualRow.createCell(i); + CellStyle cellStyle = actualRow.getCell(i - 1).getCellStyle(); + cellStyle.setAlignment(HorizontalAlignment.CENTER); + cell.setCellStyle(cellStyle); + } + return cell; + } + + + private boolean isRedactionPlaceholder(String text) { + + return prefixRedactionPlaceholders.stream() + .anyMatch(text::startsWith) || redactionPlaceholders.stream() + .anyMatch(text::contains); + } + + + private Function getFunctionForPlaceHolder(String placeholder) { + + if (placeholder.startsWith(FILE_ATTRIBUTE_PLACEHOLDER_BASE)) { + return placeholderInput -> placeholderInput.getPlaceholderModel().getFileAttributeValueByPlaceholder().get(placeholder); + } + if (placeholder.startsWith(DOSSIER_ATTRIBUTE_PLACEHOLDER_BASE)) { + return placeholderInput -> placeholderInput.getPlaceholderModel().getDossierAttributesValueByPlaceholder().get(placeholder); + } + + return switch (placeholder) { + case FILE_NAME_PLACEHOLDER -> PlaceholderInput::getFilename; + case PAGE_PLACEHOLDER -> input -> String.valueOf(input.getEntry().getPage()); + case PARAGRAPH_PLACEHOLDER -> input -> input.getEntry().getSection(); + case PARAGRAPH_INDEX_PLACEHOLDER -> input -> String.valueOf(input.getEntry().getParagraphIdx()); + case JUSTIFICATION_PLACEHOLDER -> input -> input.getEntry().getJustification(); + case JUSTIFICATION_PARAGRAPH_PLACEHOLDER, JUSTIFICATION_LEGAL_BASIS_PLACEHOLDER -> input -> input.getEntry().getJustificationParagraph(); + case JUSTIFICATION_REASON_PLACEHOLDER, JUSTIFICATION_TEXT_PLACEHOLDER -> input -> input.getEntry().getJustificationReason(); + case EXCERPT_PLACEHOLDER -> input -> input.getEntry().getExcerpt(); + case REDACTION_VALUE_PLACEHOLDER -> + input -> input.getEntry().getValue() != null ? input.getEntry().getValue().replaceAll("\n", " ").replaceAll(" {2,}", " ") : input.getEntry() + .getEntityDisplayName(); + case REDACTION_ENTITY_DISPLAY_NAME_PLACEHOLDER -> input -> input.getEntry().getEntityDisplayName(); + case SKIPPED_PLACEHOLDER -> input -> input.getEntry().isSkipped() ? "true" : "false"; + default -> input -> ""; + }; + + } + +} diff --git a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ExcelReportGenerationService.java b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ExcelReportGenerationService.java index b305d1e..d671c47 100644 --- a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ExcelReportGenerationService.java +++ b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ExcelReportGenerationService.java @@ -1,11 +1,6 @@ package com.iqser.red.service.redaction.report.v1.server.service; -import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.COMPONENT_PLACEHOLDER; -import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.DOSSIER_ATTRIBUTE_PLACEHOLDER_BASE; import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.DOSSIER_NAME_PLACEHOLDER; -import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.EXCERPT_PLACEHOLDER; -import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.FILE_ATTRIBUTES_PLACEHOLDER; -import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.FILE_ATTRIBUTE_PLACEHOLDER_BASE; import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.FILE_NAME_PLACEHOLDER; import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.FORMAT_DATE_ENG; import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.FORMAT_DATE_ENG_PLACEHOLDER; @@ -15,29 +10,10 @@ import static com.iqser.red.service.redaction.report.v1.server.service.Placehold import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.FORMAT_DATE_ISO_PLACEHOLDER; import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.FORMAT_TIME_ISO; import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.FORMAT_TIME_ISO_PLACEHOLDER; -import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.INDEX_PLACEHOLDER; -import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.JUSTIFICATION_LEGAL_BASIS_PLACEHOLDER; -import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.JUSTIFICATION_PARAGRAPH_PLACEHOLDER; -import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.JUSTIFICATION_PLACEHOLDER; -import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.JUSTIFICATION_REASON_PLACEHOLDER; -import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.JUSTIFICATION_TEXT_PLACEHOLDER; -import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.PAGE_PLACEHOLDER; -import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.PARAGRAPH_INDEX_PLACEHOLDER; -import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.PARAGRAPH_PLACEHOLDER; -import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.REDACTION_ENTITY_DISPLAY_NAME_PLACEHOLDER; -import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.REDACTION_VALUE_PLACEHOLDER; -import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.RSS_PLACEHOLDER_BASE; -import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.SCM_FUNCTION_PLACEHOLDER; -import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.SEEDS_FUNCTION_JUSTIFICATION_PLACEHOLDER; -import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.SEEDS_FUNCTION_REDACTION_GROUPED_BY_JUSTIFICATION_PAGES_PLACEHOLDER; -import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.SKIPPED_PLACEHOLDER; import java.awt.image.BufferedImage; import java.io.ByteArrayOutputStream; -import java.io.IOException; import java.time.OffsetDateTime; -import java.util.ArrayList; -import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -54,17 +30,14 @@ import org.apache.poi.ss.usermodel.Cell; import org.apache.poi.ss.usermodel.CellStyle; import org.apache.poi.ss.usermodel.ClientAnchor; import org.apache.poi.ss.usermodel.CreationHelper; +import org.apache.poi.ss.usermodel.DataFormatter; import org.apache.poi.ss.usermodel.Drawing; -import org.apache.poi.ss.usermodel.HorizontalAlignment; import org.apache.poi.ss.usermodel.Picture; -import org.apache.poi.ss.usermodel.Row; import org.apache.poi.ss.usermodel.Sheet; import org.apache.poi.xssf.streaming.SXSSFWorkbook; import org.springframework.stereotype.Service; -import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileAttributeConfig; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel; -import com.iqser.red.service.redaction.report.v1.server.client.FileAttributesConfigClient; import com.iqser.red.service.redaction.report.v1.server.model.CellIdentifier; import com.iqser.red.service.redaction.report.v1.server.model.ExcelModel; import com.iqser.red.service.redaction.report.v1.server.model.ImagePlaceholder; @@ -84,31 +57,10 @@ import lombok.extern.slf4j.Slf4j; @RequiredArgsConstructor public class ExcelReportGenerationService { - private static final Set prefixRedactionPlaceholders = Set.of(DOSSIER_ATTRIBUTE_PLACEHOLDER_BASE, FILE_ATTRIBUTE_PLACEHOLDER_BASE); - private static final Set redactionPlaceholders = Set.of(FILE_NAME_PLACEHOLDER, - PAGE_PLACEHOLDER, - PARAGRAPH_PLACEHOLDER, - PARAGRAPH_INDEX_PLACEHOLDER, - JUSTIFICATION_PLACEHOLDER, - EXCERPT_PLACEHOLDER, - JUSTIFICATION_PARAGRAPH_PLACEHOLDER, - JUSTIFICATION_REASON_PLACEHOLDER, - REDACTION_VALUE_PLACEHOLDER, - JUSTIFICATION_LEGAL_BASIS_PLACEHOLDER, - JUSTIFICATION_TEXT_PLACEHOLDER, - SEEDS_FUNCTION_REDACTION_GROUPED_BY_JUSTIFICATION_PAGES_PLACEHOLDER, - SEEDS_FUNCTION_JUSTIFICATION_PLACEHOLDER, - REDACTION_ENTITY_DISPLAY_NAME_PLACEHOLDER, - SCM_FUNCTION_PLACEHOLDER, - SKIPPED_PLACEHOLDER, - FILE_ATTRIBUTES_PLACEHOLDER, - INDEX_PLACEHOLDER, - COMPONENT_PLACEHOLDER); - private final ScmReportService componentReportService; - private final FileAttributesConfigClient fileAttributesConfigClient; private final ComponentRowsReportService componentRowsReportService; private CreationHelper creationHelper; + final DataFormatter formatter = new DataFormatter(); @Timed("redactmanager_generateExcelReport") @@ -127,69 +79,72 @@ public class ExcelReportGenerationService { .filter(entry -> !entry.isSkipped()) .collect(Collectors.toList()) : allReportEntries; - try { + for (Sheet sheet : workbook) { - for (Sheet sheet : workbook) { + if (!excelModel.isRowsBeforeRedactionEntryRowsAdded()) { - if (!excelModel.isRowsBeforeRedactionEntryRowsAdded()) { - - for (Map.Entry columnWidthEntry : excelModel.getCellWidths().entrySet()) { - sheet.setColumnWidth(columnWidthEntry.getKey(), columnWidthEntry.getValue()); - } - - addRows(workbook, - sheet, - excelModel.getCellsToCopyBeforeRedactionPlaceholderRow(), - excelModel.isRssPlaceholdersPresent() ? excelModel.getWrittenRows().size() : 0, - placeholderModel, - dossierName, - fileModel.getFilename(), - excelModel); - - if (!excelModel.isRssPlaceholdersPresent()) { - excelModel.setRowsBeforeRedactionEntryRowsAdded(true); - } + for (Map.Entry columnWidthEntry : excelModel.getCellWidths().entrySet()) { + sheet.setColumnWidth(columnWidthEntry.getKey(), columnWidthEntry.getValue()); } - if (!excelModel.isRssPlaceholdersPresent() && !excelModel.isFileAttributesPlaceholderPresent() && !excelModel.isScmFunctionPlaceholderPresent()) { - addRedactionEntryRows(sheet, reportEntries, fileModel.getFilename(), excelModel, placeholderModel); - } + addRows(workbook, + sheet, + excelModel.getCellsToCopyBeforeRedactionPlaceholderRow(), + excelModel.isRssPlaceholdersPresent() ? excelModel.getWrittenRows().size() : 0, + placeholderModel, + dossierName, + fileModel.getFilename(), + excelModel); - if (excelModel.isFileAttributesPlaceholderPresent()) { - componentReportService.addScmRows(sheet, fileModel, excelModel); + if (!excelModel.isRssPlaceholdersPresent()) { + excelModel.setRowsBeforeRedactionEntryRowsAdded(true); + } + } + + if (isRedactionReport(excelModel)) { + addRedactionEntryRows(sheet, reportEntries, fileModel.getFilename(), excelModel, placeholderModel); + } else { + if (excelModel.getComponentReport().isComponentReport()) { + componentReportService.addComponentRows(sheet, fileModel, excelModel); } if (excelModel.isScmFunctionPlaceholderPresent()) { componentRowsReportService.addComponentRows(sheet, fileModel, excelModel); } - - if (isLastFile) { - addRows(workbook, - sheet, - excelModel.getCellsToCopyAfterRedactionPlaceholderRow(), - excelModel.getWrittenRows().size(), - placeholderModel, - dossierName, - fileModel.getFilename(), - excelModel); - } } - log.info("Report Generation took: {} for file with id {}, pageCount: {}, entityLogEntryCount: {}, reportName: {}, className: {}", - System.currentTimeMillis() - start, - fileModel.getId(), - fileModel.getNumberOfPages(), - reportEntries.size(), - reportTemplateName, - getClass().getSimpleName()); - - } catch (Exception e) { - throw new RuntimeException(e); + if (isLastFile) { + addRows(workbook, + sheet, + excelModel.getCellsToCopyAfterRedactionPlaceholderRow(), + excelModel.getWrittenRows().size(), + placeholderModel, + dossierName, + fileModel.getFilename(), + excelModel); + } } + + log.info("Report Generation took: {} for file with id {}, pageCount: {}, entityLogEntryCount: {}, reportName: {}, className: {}", + System.currentTimeMillis() - start, + fileModel.getId(), + fileModel.getNumberOfPages(), + reportEntries.size(), + reportTemplateName, + getClass().getSimpleName()); + + } + + + private static boolean isRedactionReport(ExcelModel excelModel) { + + return !(excelModel.isRssPlaceholdersPresent() + || excelModel.isFileAttributesPlaceholderPresent() + || excelModel.isScmFunctionPlaceholderPresent() + || excelModel.getComponentReport().isComponentReport()); } - @SneakyThrows private void addRows(SXSSFWorkbook workbook, Sheet sheet, Map copiedCells, @@ -208,6 +163,11 @@ public class ExcelReportGenerationService { continue; } + log.debug("Copying cell {},{} with content {} from template", + cellsToCopyEntry.getKey().getRowIndex(), + cellsToCopyEntry.getKey().getColumnIndex(), + formatter.formatCellValue(cellsToCopyEntry.getValue())); + int indexToAddRow = cellsToCopyEntry.getKey().getRowIndex() + numberOfRowsToShift - skipRows; if (!createdCopyRows.contains(indexToAddRow)) { @@ -241,12 +201,18 @@ public class ExcelReportGenerationService { Map> placeholderCellPos = excelModel.getPlaceholderCellPos(); reportEntries.forEach(entry -> { + PlaceholderInput placeholderInput = new PlaceholderInput(filename, entry, placeholderModel, null); sheet.createRow(rowIndex.get()); excelModel.getWrittenRows().add(rowIndex.get()); - for (Map.Entry> entry1 : placeholderCellPos.entrySet()) { - Cell cell = sheet.getRow(rowIndex.get()).createCell(entry1.getKey()); - cell.setCellValue(entry1.getValue().apply(new PlaceholderInput(filename, entry, placeholderModel, null)) == null ? "" : entry1.getValue() - .apply(new PlaceholderInput(filename, entry, placeholderModel, null))); + for (Map.Entry> colIdxToCellValueTransform : placeholderCellPos.entrySet()) { + Cell cell = sheet.getRow(rowIndex.get()).createCell(colIdxToCellValueTransform.getKey()); + + String cellValue = colIdxToCellValueTransform.getValue().apply(placeholderInput); + + if (cellValue == null) { + cellValue = ""; + } + cell.setCellValue(cellValue); } rowIndex.getAndIncrement(); }); @@ -268,7 +234,8 @@ public class ExcelReportGenerationService { } - private void replacePlaceholdersForImagePlaceholder(SXSSFWorkbook workbook, Sheet sheet, Cell cell, PlaceholderModel placeholderModel) throws IOException { + @SneakyThrows + private void replacePlaceholdersForImagePlaceholder(SXSSFWorkbook workbook, Sheet sheet, Cell cell, PlaceholderModel placeholderModel) { for (ImagePlaceholder imagePlaceholder : placeholderModel.getImagePlaceholders()) { @@ -335,18 +302,15 @@ public class ExcelReportGenerationService { return filename; } if (placeholderModel.getFileAttributeValueByPlaceholder().containsKey(placeholder)) { - return placeholderModel.getFileAttributeValueByPlaceholder() - .get(placeholder); + return placeholderModel.getFileAttributeValueByPlaceholder().get(placeholder); } if (placeholderModel.getDossierAttributesValueByPlaceholder().containsKey(placeholder)) { - return placeholderModel.getDossierAttributesValueByPlaceholder() - .get(placeholder); + return placeholderModel.getDossierAttributesValueByPlaceholder().get(placeholder); } if (placeholderModel.getRssComponentPlaceholder() != null && placeholderModel.getRssComponentPlaceholder().containsKey(placeholder)) { - return placeholderModel.getRssComponentPlaceholder() - .get(placeholder); + return placeholderModel.getRssComponentPlaceholder().get(placeholder); } return null; @@ -383,167 +347,6 @@ public class ExcelReportGenerationService { } - @Timed("redactmanager_calculateExcelModel") - @SuppressWarnings("checkstyle:all") - public ExcelModel calculateExcelModel(Sheet sheet, String dossierTemplateId) { - - long start = System.currentTimeMillis(); - - Map> placeholderCellPos = new HashMap<>(); - Map columnWidths = new HashMap<>(); - Map cellsToCopyBeforePlaceholderRow = new HashMap<>(); - Map cellsToCopyAfterPlaceholderRow = new HashMap<>(); - Map cellsToCopyInPlaceholderRow = new HashMap<>(); - - boolean hasRssPlaceHolders = false; - boolean hasSkippedPlaceholder = false; - int placeholderRow = -1; - boolean placeholderInFirstRow = false; - boolean fileAttributesPlaceholder = false; - boolean hasScmFunctionPlaceholder = false; - - for (int j = 0; j < sheet.getLastRowNum() + 1; j++) { - Row actualRow = sheet.getRow(j); - if (actualRow != null) { - int lastCellIndex = actualRow.getLastCellNum(); - for (int i = 0; i < lastCellIndex; i++) { - Cell cell = sheet.getRow(j).getCell(i); - if (cell != null) { - - boolean skipCopy = false; - int columnWidth = sheet.getColumnWidth(i); - columnWidths.put(i, columnWidth); - String cellStringValue = cell.getStringCellValue(); - - if (j == 0 && cellStringValue.contains("{{")) { - placeholderInFirstRow = true; - } - - if (cellStringValue.contains(RSS_PLACEHOLDER_BASE)) { - hasRssPlaceHolders = true; - } - - if (cellStringValue.contains(SCM_FUNCTION_PLACEHOLDER)) { - hasScmFunctionPlaceholder = true; - } - - if (cellStringValue.contains(FILE_ATTRIBUTES_PLACEHOLDER)) { - fileAttributesPlaceholder = true; - - // For each file attribute we have to replace the column header with the corresponding file attribute. - List fileAttributeConfigs = fileAttributesConfigClient.getFileAttributeConfigs(dossierTemplateId); - var iterator = fileAttributeConfigs.iterator(); - - while (iterator.hasNext()) { - - cell = sheet.getRow(j).getCell(i); - - // If there is more than one file attribute, starting from the 2nd one we won't find a cell in the given index so we have to create it. - if (cell == null) { - cell = sheet.getRow(j).createCell(i); - CellStyle cellStyle = sheet.getRow(j).getCell(i - 1).getCellStyle(); - cellStyle.setAlignment(HorizontalAlignment.CENTER); - cell.setCellStyle(cellStyle); - } - - cell.setCellValue(iterator.next().getLabel()); - cellsToCopyBeforePlaceholderRow.put(new CellIdentifier(j, i), cell); - - // If there is more than one file attribute we have to insert a new column, so we shift all columns to the right by 1 - // and increase the current and last cell index. - if (iterator.hasNext()) { - lastCellIndex += 1; - i += 1; - sheet.shiftColumns(i, lastCellIndex, 1); - } - } - - skipCopy = true; - } - - if (!skipCopy) { - if (isRedactionPlaceholder(cellStringValue)) { - if (cellStringValue.equals(SKIPPED_PLACEHOLDER)) { - hasSkippedPlaceholder = true; - } - placeholderCellPos.put(i, getFunctionForPlaceHolder(cellStringValue)); - placeholderRow = j; - cellsToCopyInPlaceholderRow.put(new CellIdentifier(j, i), cell); - } else { - if (placeholderRow == -1) { - cellsToCopyBeforePlaceholderRow.put(new CellIdentifier(j, i), cell); - } else { - cellsToCopyAfterPlaceholderRow.put(new CellIdentifier(j, i), cell); - } - } - } - } - } - } - } - - if (hasRssPlaceHolders) { - cellsToCopyBeforePlaceholderRow.putAll(cellsToCopyInPlaceholderRow); - cellsToCopyBeforePlaceholderRow.putAll(cellsToCopyAfterPlaceholderRow); - cellsToCopyAfterPlaceholderRow = new HashMap<>(); - } - - log.debug("Calculate Placeholder Cells took: {}", System.currentTimeMillis() - start); - return new ExcelModel(placeholderCellPos, - placeholderRow, - columnWidths, - cellsToCopyBeforePlaceholderRow, - cellsToCopyAfterPlaceholderRow, - new ArrayList<>(), - false, - hasRssPlaceHolders, - placeholderInFirstRow, - false, - hasSkippedPlaceholder, - fileAttributesPlaceholder, - hasScmFunctionPlaceholder); - } - - - private boolean isRedactionPlaceholder(String text) { - - return prefixRedactionPlaceholders.stream() - .anyMatch(text::startsWith) || redactionPlaceholders.stream() - .anyMatch(text::contains); - } - - - private Function getFunctionForPlaceHolder(String placeholder) { - - if (placeholder.startsWith(FILE_ATTRIBUTE_PLACEHOLDER_BASE)) { - return placeholderInput -> placeholderInput.getPlaceholderModel().getFileAttributeValueByPlaceholder() - .get(placeholder); - } - if (placeholder.startsWith(DOSSIER_ATTRIBUTE_PLACEHOLDER_BASE)) { - return placeholderInput -> placeholderInput.getPlaceholderModel().getDossierAttributesValueByPlaceholder() - .get(placeholder); - } - - return switch (placeholder) { - case FILE_NAME_PLACEHOLDER -> PlaceholderInput::getFilename; - case PAGE_PLACEHOLDER -> input -> String.valueOf(input.getEntry().getPage()); - case PARAGRAPH_PLACEHOLDER -> input -> input.getEntry().getSection(); - case PARAGRAPH_INDEX_PLACEHOLDER -> input -> String.valueOf(input.getEntry().getParagraphIdx()); - case JUSTIFICATION_PLACEHOLDER -> input -> input.getEntry().getJustification(); - case JUSTIFICATION_PARAGRAPH_PLACEHOLDER, JUSTIFICATION_LEGAL_BASIS_PLACEHOLDER -> input -> input.getEntry().getJustificationParagraph(); - case JUSTIFICATION_REASON_PLACEHOLDER, JUSTIFICATION_TEXT_PLACEHOLDER -> input -> input.getEntry().getJustificationReason(); - case EXCERPT_PLACEHOLDER -> input -> input.getEntry().getExcerpt(); - case REDACTION_VALUE_PLACEHOLDER -> - input -> input.getEntry().getValue() != null ? input.getEntry().getValue().replaceAll("\n", " ").replaceAll(" {2,}", " ") : input.getEntry() - .getEntityDisplayName(); - case REDACTION_ENTITY_DISPLAY_NAME_PLACEHOLDER -> input -> input.getEntry().getEntityDisplayName(); - case SKIPPED_PLACEHOLDER -> input -> input.getEntry().isSkipped() ? "true" : "false"; - default -> input -> ""; - }; - - } - - @SneakyThrows public byte[] toByteArray(SXSSFWorkbook workbook) { diff --git a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ExcelReportTemplateService.java b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ExcelReportTemplateService.java index eb03bd6..a6ddb56 100644 --- a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ExcelReportTemplateService.java +++ b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ExcelReportTemplateService.java @@ -15,13 +15,13 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel; import com.iqser.red.service.redaction.report.v1.api.model.ReportType; import com.iqser.red.service.redaction.report.v1.api.model.StoredFileInformation; +import com.iqser.red.service.redaction.report.v1.server.cache.ReportTemplateCache; +import com.iqser.red.service.redaction.report.v1.server.model.ExcelModel; import com.iqser.red.service.redaction.report.v1.server.model.MultiFileWorkbook; import com.iqser.red.service.redaction.report.v1.server.model.PlaceholderModel; import com.iqser.red.service.redaction.report.v1.server.model.ReportRedactionEntry; import com.iqser.red.service.redaction.report.v1.server.model.ReportTemplatesModel; -import com.iqser.red.service.redaction.report.v1.server.storage.ReportStorageService; import com.iqser.red.service.redaction.report.v1.server.storage.ReportStorageServiceAsyncWrapper; -import com.iqser.red.service.redaction.report.v1.server.cache.ReportTemplateCache; import lombok.AccessLevel; import lombok.RequiredArgsConstructor; @@ -33,11 +33,11 @@ import lombok.experimental.FieldDefaults; @SuppressWarnings("PMD") public class ExcelReportTemplateService { - ReportStorageService reportStorageService; ExcelReportGenerationService excelTemplateReportGenerationService; GeneratePlaceholderService generatePlaceholderService; ReportStorageServiceAsyncWrapper reportStorageServiceAsyncWrapper; ReportTemplateCache reportTemplateCache; + ExcelModelFactory excelModelFactory; public CompletableFuture createExcelReportFromTemplateAsync(Dossier dossier, @@ -55,7 +55,7 @@ public class ExcelReportTemplateService { for (Sheet sheet : readWorkbook) { writeWorkbook.createSheet(sheet.getSheetName()); } - var excelModel = excelTemplateReportGenerationService.calculateExcelModel(readWorkbook.getSheetAt(0), dossier.getDossierTemplateId()); + var excelModel = excelModelFactory.calculateExcelModel(readWorkbook.getSheetAt(0), dossier.getDossierTemplateId()); if (excelModel.isRssPlaceholdersPresent()) { generatePlaceholderService.resolveRssValues(fileStatus, placeholderModel); } @@ -88,11 +88,10 @@ public class ExcelReportTemplateService { for (Sheet sheet : readWorkbook) { writeWorkbook.createSheet(sheet.getSheetName()); } - MultiFileWorkbook multiFileWorkbook = new MultiFileWorkbook(readWorkbook, - writeWorkbook, - templateId, - reportTemplate.getFileName(), - excelTemplateReportGenerationService.calculateExcelModel(readWorkbook.getSheetAt(0), dossierTemplateId)); + + ExcelModel excelModel = excelModelFactory.calculateExcelModel(readWorkbook.getSheetAt(0), dossierTemplateId); + + MultiFileWorkbook multiFileWorkbook = new MultiFileWorkbook(readWorkbook, writeWorkbook, templateId, reportTemplate.getFileName(), excelModel); reportTemplatesModel.multiFileWorkbookReportTemplates.add(multiFileWorkbook); } catch (IOException e) { throw new RuntimeException("Could not generate multifile excel report."); diff --git a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/GeneratePlaceholderService.java b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/GeneratePlaceholderService.java index 3dfd01b..6840ff4 100644 --- a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/GeneratePlaceholderService.java +++ b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/GeneratePlaceholderService.java @@ -7,7 +7,7 @@ import static com.iqser.red.service.redaction.report.v1.server.service.Placehold import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.FORMAT_DATE_ISO_PLACEHOLDER; import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.FORMAT_TIME_ISO_PLACEHOLDER; import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.IUCLID_FUNCTION_PLACEHOLDER; -import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.RSS_PLACEHOLDER_BASE; +import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.COMPONENT_PLACEHOLDER_BASE; import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.SEEDS_FUNCTION_JUSTIFICATION_PLACEHOLDER; import static com.iqser.red.service.redaction.report.v1.server.service.PlaceholderService.SEEDS_FUNCTION_REDACTION_GROUPED_BY_JUSTIFICATION_PAGES_PLACEHOLDER; @@ -131,7 +131,7 @@ public class GeneratePlaceholderService { if (!rssResponse.getFiles().isEmpty()) { for (Map.Entry rssEntry : rssResponse.getFiles().get(0).getResult().entrySet()) { - rssPlaceholders.put(RSS_PLACEHOLDER_BASE + rssEntry.getKey() + "}}", rssEntry.getValue().getOriginalValue()); + rssPlaceholders.put(COMPONENT_PLACEHOLDER_BASE + rssEntry.getKey() + "}}", rssEntry.getValue().getOriginalValue()); } } diff --git a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/PlaceholderService.java b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/PlaceholderService.java index 4db3f4f..b167d96 100644 --- a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/PlaceholderService.java +++ b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/PlaceholderService.java @@ -53,7 +53,7 @@ public class PlaceholderService { public static final String SCM_FUNCTION_PLACEHOLDER = "{{function.scm}}"; public static final String FILE_ATTRIBUTE_PLACEHOLDER_BASE = "{{file.attribute."; public static final String DOSSIER_ATTRIBUTE_PLACEHOLDER_BASE = "{{dossier.attribute."; - public static final String RSS_PLACEHOLDER_BASE = "{{component."; + public static final String COMPONENT_PLACEHOLDER_BASE = "{{component."; public static final String COMPONENT_PLACEHOLDER = "{{component}}"; public static final String FILE_ATTRIBUTES_PLACEHOLDER = "{{file.attributes}}"; public static final String INDEX_PLACEHOLDER = "{{index}}"; diff --git a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ReportGenerationService.java b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ReportGenerationService.java index e90d8f9..42b2737 100644 --- a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ReportGenerationService.java +++ b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ReportGenerationService.java @@ -30,7 +30,6 @@ import com.iqser.red.service.redaction.report.v1.server.storage.ReportStorageSer import io.micrometer.core.annotation.Timed; import lombok.AccessLevel; import lombok.RequiredArgsConstructor; -import lombok.SneakyThrows; import lombok.experimental.FieldDefaults; import lombok.extern.slf4j.Slf4j; @@ -53,7 +52,6 @@ public class ReportGenerationService { ReportTemplateService reportTemplateService; - @SneakyThrows @Timed("redactmanager_generateReports") public String generateReports(ReportRequestMessage reportMessage) { @@ -71,9 +69,8 @@ public class ReportGenerationService { var isLastFile = fileIdIndex == reportMessage.getFileIds().size() - 1; String dossierId = dossier.getId(); - String fileId = reportMessage.getFileIds() - .get(fileIdIndex); - String dossierName = dossier.getDossierName(); + String fileId = reportMessage.getFileIds().get(fileIdIndex); + String dossierName = dossier.getName(); var fileStatus = fileStatusClient.getFileStatus(dossierId, fileId); generatePlaceholderService.resolveFileAttributeValues(fileStatus, placeholderModel); @@ -218,12 +215,12 @@ public class ReportGenerationService { return CompletableFuture.allOf(singleFilesTemplates.stream() .map(reportTemplate -> reportTemplateService.createReportFromTemplateAsync(dossier, - fileStatus, - placeholderModel, - reportTemplate.getFileName(), - downloadId, - reportEntries, - reportTemplate).thenAccept(storedFileInformation::add)) + fileStatus, + placeholderModel, + reportTemplate.getFileName(), + downloadId, + reportEntries, + reportTemplate).thenAccept(storedFileInformation::add)) .toArray(CompletableFuture[]::new)); } diff --git a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ScmReportService.java b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ScmReportService.java index 2d376a6..29f697c 100644 --- a/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ScmReportService.java +++ b/redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ScmReportService.java @@ -6,7 +6,13 @@ import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; -import org.apache.poi.ss.usermodel.*; +import org.apache.poi.ss.usermodel.Cell; +import org.apache.poi.ss.usermodel.CellStyle; +import org.apache.poi.ss.usermodel.DataFormatter; +import org.apache.poi.ss.usermodel.HorizontalAlignment; +import org.apache.poi.ss.usermodel.Row; +import org.apache.poi.ss.usermodel.Sheet; +import org.apache.poi.ss.usermodel.VerticalAlignment; import org.apache.poi.ss.util.CellUtil; import org.apache.poi.xssf.streaming.SXSSFSheet; import org.springframework.stereotype.Service; @@ -30,44 +36,20 @@ import lombok.experimental.FieldDefaults; @SuppressWarnings("checkstyle:all") public class ScmReportService { - public static final int MAX_CELL_TEXT_LENGTH = 32767; final ComponentClient componentResource; final FileAttributesConfigClient fileAttributesClient; final CellTextChunkingService cellTextChunkingService; final DataFormatter formatter = new DataFormatter(); - public void addScmRows(Sheet sheet, FileModel fileModel, ExcelModel excelModel) { - - var firstRow = sheet.getRow(0); - - int firstRowIndex = 1; - while (firstRow == null) { - firstRow = sheet.getRow(firstRowIndex); - firstRowIndex++; - } - - // Ensure we have the correct first row - Cell firstCell = firstRow.getCell(0); - if (!formatter.formatCellValue(firstCell).equals("File")) { - firstRow = sheet.getRow(0); - } - - int fileColumnIndex = 0; - int textColumnIndex = firstRow.getLastCellNum() - 1; - int valueIndexColumn = firstRow.getLastCellNum() - 2; - int componentColumnIndex = firstRow.getLastCellNum() - 3; - - CellUtil.setAlignment(firstRow.getCell(textColumnIndex), HorizontalAlignment.LEFT); + public void addComponentRows(Sheet sheet, FileModel fileModel, ExcelModel excelModel) { ComponentLog componentLog = componentResource.getComponentLog(fileModel.getDossierId(), fileModel.getId(), true); - AtomicInteger rowIndex = new AtomicInteger(excelModel.getRedactionPlaceholderRow()); + AtomicInteger rowIndexCounter = new AtomicInteger(excelModel.getRedactionPlaceholderRow()); Map componentIndexMap = new HashMap<>(); - Row finalFirstRow = firstRow; - componentLog.getComponentLogEntries() .forEach(componentLogEntry -> { @@ -76,51 +58,44 @@ public class ScmReportService { } // Process each ComponentLogEntryValue - addOtherCells(sheet, - excelModel, - rowIndex, - componentIndexMap, - componentLogEntry, - fileModel, - finalFirstRow, - fileColumnIndex, - componentColumnIndex, - valueIndexColumn, - textColumnIndex); + addOtherCells(sheet, excelModel, rowIndexCounter, componentIndexMap, componentLogEntry, fileModel); }); + autosizeColumns(sheet, rowIndexCounter); + + sheet.createRow(rowIndexCounter.get()); + excelModel.getWrittenRows().add(rowIndexCounter.get()); + excelModel.setRedactionPlaceholderRow(rowIndexCounter.getAndIncrement()); + } + + + private static void autosizeColumns(Sheet sheet, AtomicInteger rowIndexCounter) { // Autosize all cells besides the last column because the extracted text should be multiline if (sheet instanceof SXSSFSheet) { ((SXSSFSheet) sheet).trackAllColumnsForAutoSizing(); - for (int i = 0; i < firstRow.getLastCellNum() - 1; i++) { - sheet.autoSizeColumn(i); + Row row = sheet.getRow(rowIndexCounter.get() - 1); + if (row != null) { + for (int i = 0; i < row.getLastCellNum(); i++) { + sheet.autoSizeColumn(i); + } } } - - sheet.createRow(rowIndex.get()); - excelModel.getWrittenRows().add(rowIndex.get()); - excelModel.setRedactionPlaceholderRow(rowIndex.getAndIncrement()); } private void addOtherCells(Sheet sheet, ExcelModel excelModel, - AtomicInteger rowIndex, + AtomicInteger rowIndexCounter, Map componentIndexMap, ComponentLogEntry componentLogEntry, - FileModel fileModel, - Row firstRow, - int fileColumnIndex, - int componentColumnIndex, - int valueIndexColumn, - int textColumnIndex) { + FileModel fileModel) { String componentValue = componentLogEntry.getName().replaceAll("_", " "); int componentIndex = componentIndexMap.getOrDefault(componentValue, 0); - List fileAttributeModels = buildFileAttributeModels(fileModel, firstRow); + List fileAttributeModels = excelModel.getFileAttributeColumns(); for (ComponentLogEntryValue componentLogEntryValue : componentLogEntry.getComponentValues()) { @@ -130,23 +105,24 @@ public class ScmReportService { List textChunks = cellTextChunkingService.process(componentLogEntryValue.getValue()); for (String chunk : textChunks) { - int rowIndexAndIncrement = rowIndex.getAndIncrement(); - Row row = sheet.createRow(rowIndexAndIncrement); - excelModel.getWrittenRows().add(rowIndexAndIncrement); + int currentRowIdx = rowIndexCounter.getAndIncrement(); + Row row = sheet.createRow(currentRowIdx); + excelModel.getWrittenRows().add(currentRowIdx); - Cell fileNameCell = row.createCell(fileColumnIndex); + Cell fileNameCell = row.createCell(0); fileNameCell.setCellValue(fileModel.getFilename() == null ? "" : fileModel.getFilename()); CellUtil.setVerticalAlignment(fileNameCell, VerticalAlignment.TOP); - Cell componentNameCell = row.createCell(componentColumnIndex); + Cell componentNameCell = row.createCell(excelModel.getComponentReport().getComponentNameColumn()); componentNameCell.setCellValue(componentValue); + if (excelModel.getComponentReport().writeComponentValueIndices()) { + Cell indexCell = row.createCell(excelModel.getComponentReport().getComponentValueIndexColumn()); + indexCell.setCellValue(componentIndex); + CellUtil.setAlignment(indexCell, HorizontalAlignment.CENTER); + CellUtil.setVerticalAlignment(indexCell, VerticalAlignment.TOP); + } - Cell indexCell = row.createCell(valueIndexColumn); - indexCell.setCellValue(componentIndex); - CellUtil.setAlignment(indexCell, HorizontalAlignment.CENTER); - CellUtil.setVerticalAlignment(indexCell, VerticalAlignment.TOP); - - Cell textCell = row.createCell(textColumnIndex); + Cell textCell = row.createCell(excelModel.getComponentReport().getComponentValueColumn()); textCell.setCellValue(chunk); CellStyle cellStyle = sheet.getWorkbook().createCellStyle(); cellStyle.setWrapText(true); @@ -162,8 +138,7 @@ public class ScmReportService { for (FileAttributeModel fileAttributeModel : fileAttributeModels) { Cell fileAttributeCell = row.createCell(fileAttributeModel.getCell().getColumnIndex()); - String cellValue = fileModel.getFileAttributes() - .get(fileAttributeModel.getId()); + String cellValue = fileModel.getFileAttributes().get(fileAttributeModel.getId()); fileAttributeCell.setCellValue(cellValue == null ? "" : cellValue); CellUtil.setAlignment(fileAttributeCell, HorizontalAlignment.CENTER); CellUtil.setVerticalAlignment(fileAttributeCell, VerticalAlignment.TOP); diff --git a/redaction-report-service-v1/redaction-report-service-server-v1/src/test/java/com/iqser/red/service/redaction/report/v1/server/RedactionReportIntegrationTest.java b/redaction-report-service-v1/redaction-report-service-server-v1/src/test/java/com/iqser/red/service/redaction/report/v1/server/RedactionReportIntegrationTest.java index ea8a81a..4aba93e 100644 --- a/redaction-report-service-v1/redaction-report-service-server-v1/src/test/java/com/iqser/red/service/redaction/report/v1/server/RedactionReportIntegrationTest.java +++ b/redaction-report-service-v1/redaction-report-service-server-v1/src/test/java/com/iqser/red/service/redaction/report/v1/server/RedactionReportIntegrationTest.java @@ -73,6 +73,7 @@ import com.iqser.red.service.redaction.report.v1.server.model.ImagePlaceholder; import com.iqser.red.service.redaction.report.v1.server.model.PlaceholderModel; import com.iqser.red.service.redaction.report.v1.server.model.ReportRedactionEntry; import com.iqser.red.service.redaction.report.v1.server.service.EntityLogConverterService; +import com.iqser.red.service.redaction.report.v1.server.service.ExcelModelFactory; import com.iqser.red.service.redaction.report.v1.server.service.ExcelReportGenerationService; import com.iqser.red.service.redaction.report.v1.server.service.GeneratePlaceholderService; import com.iqser.red.service.redaction.report.v1.server.service.WordReportGenerationService; @@ -135,6 +136,9 @@ public class RedactionReportIntegrationTest { @Autowired private GeneratePlaceholderService generatePlaceholderService; + @Autowired + private ExcelModelFactory excelModelFactory; + @MockBean private DictionaryClient dictionaryClient; @@ -423,7 +427,7 @@ public class RedactionReportIntegrationTest { var placeholders = buildPlaceHolderModel(new HashMap<>(), new HashMap<>(), List.of()); XSSFWorkbook readWorkbook = new XSSFWorkbook(templateResource.getInputStream()); - var excelModel = excelTemplateReportGenerationService.calculateExcelModel(readWorkbook.getSheetAt(0), dossier.getDossierTemplateId()); + var excelModel = excelModelFactory.calculateExcelModel(readWorkbook.getSheetAt(0), dossier.getDossierTemplateId()); SXSSFWorkbook writeWorkbook = new SXSSFWorkbook(); writeWorkbook.createSheet("Sheet1"); excelTemplateReportGenerationService.generateExcelReport(reportEntries, placeholders, "test", writeWorkbook, dossier.getName(), fileModel, excelModel, true); @@ -454,7 +458,7 @@ public class RedactionReportIntegrationTest { var placeholders = buildPlaceHolderModel(new HashMap<>(), new HashMap<>(), List.of()); XSSFWorkbook readWorkbook = new XSSFWorkbook(templateResource.getInputStream()); - var excelModel = excelTemplateReportGenerationService.calculateExcelModel(readWorkbook.getSheetAt(0), dossier.getDossierTemplateId()); + var excelModel = excelModelFactory.calculateExcelModel(readWorkbook.getSheetAt(0), dossier.getDossierTemplateId()); SXSSFWorkbook writeWorkbook = new SXSSFWorkbook(); writeWorkbook.createSheet("Sheet1"); excelTemplateReportGenerationService.generateExcelReport(reportEntries, placeholders, "test", writeWorkbook, "dossierName", fileModel, excelModel, false); @@ -500,7 +504,7 @@ public class RedactionReportIntegrationTest { var placeholders = buildPlaceHolderModel(new HashMap<>(), new HashMap<>(), List.of()); XSSFWorkbook readWorkbook = new XSSFWorkbook(templateResource.getInputStream()); - var excelModel = excelTemplateReportGenerationService.calculateExcelModel(readWorkbook.getSheetAt(0), dossier.getDossierTemplateId()); + var excelModel = excelModelFactory.calculateExcelModel(readWorkbook.getSheetAt(0), dossier.getDossierTemplateId()); SXSSFWorkbook writeWorkbook = new SXSSFWorkbook(); writeWorkbook.createSheet("Sheet1"); excelTemplateReportGenerationService.generateExcelReport(reportEntries, placeholders, "test", writeWorkbook, dossier.getName(), fileModel, excelModel, true); @@ -534,7 +538,7 @@ public class RedactionReportIntegrationTest { List.of(new ImagePlaceholder("{{dossier.attribute.Signature}}", IOUtils.toByteArray(imageResource.getInputStream())))); XSSFWorkbook readWorkbook = new XSSFWorkbook(templateResource.getInputStream()); - var excelModel = excelTemplateReportGenerationService.calculateExcelModel(readWorkbook.getSheetAt(0), dossier.getDossierTemplateId()); + var excelModel = excelModelFactory.calculateExcelModel(readWorkbook.getSheetAt(0), dossier.getDossierTemplateId()); SXSSFWorkbook writeWorkbook = new SXSSFWorkbook(); writeWorkbook.createSheet("Sheet1"); excelTemplateReportGenerationService.generateExcelReport(reportEntries, placeholders, "test", writeWorkbook, "dossierName", fileModel, excelModel, true); @@ -568,7 +572,7 @@ public class RedactionReportIntegrationTest { IOUtils.toByteArray(imageResource.getInputStream())))); XSSFWorkbook readWorkbook = new XSSFWorkbook(templateResource.getInputStream()); - var excelModel = excelTemplateReportGenerationService.calculateExcelModel(readWorkbook.getSheetAt(0), dossier.getDossierTemplateId()); + var excelModel = excelModelFactory.calculateExcelModel(readWorkbook.getSheetAt(0), dossier.getDossierTemplateId()); SXSSFWorkbook writeWorkbook = new SXSSFWorkbook(); writeWorkbook.createSheet("Sheet1"); excelTemplateReportGenerationService.generateExcelReport(reportEntries, placeholders, "test", writeWorkbook, "dossierName", fileModel, excelModel, true); @@ -609,7 +613,7 @@ public class RedactionReportIntegrationTest { var placeholders = buildPlaceHolderModel(new HashMap<>(), Map.of("{{file.attribute.placeholder}}", "OECD Number"), List.of()); XSSFWorkbook readWorkbook = new XSSFWorkbook(templateResource.getInputStream()); - var excelModel = excelTemplateReportGenerationService.calculateExcelModel(readWorkbook.getSheetAt(0), dossier.getDossierTemplateId()); + var excelModel = excelModelFactory.calculateExcelModel(readWorkbook.getSheetAt(0), dossier.getDossierTemplateId()); SXSSFWorkbook writeWorkbook = new SXSSFWorkbook(); writeWorkbook.createSheet("Sheet1"); excelTemplateReportGenerationService.generateExcelReport(reportEntries, placeholders, "test", writeWorkbook, dossier.getName(), fileModel, excelModel, true); @@ -647,7 +651,7 @@ public class RedactionReportIntegrationTest { ClassPathResource templateResource = new ClassPathResource("templates/scm_report.xlsx"); XSSFWorkbook readWorkbook = new XSSFWorkbook(templateResource.getInputStream()); - var excelModel = excelTemplateReportGenerationService.calculateExcelModel(readWorkbook.getSheetAt(0), dossier.getDossierTemplateId()); + var excelModel = excelModelFactory.calculateExcelModel(readWorkbook.getSheetAt(0), dossier.getDossierTemplateId()); SXSSFWorkbook writeWorkbook = new SXSSFWorkbook(); writeWorkbook.createSheet("Sheet1"); diff --git a/redaction-report-service-v1/redaction-report-service-server-v1/src/test/resources/logback-spring.xml b/redaction-report-service-v1/redaction-report-service-server-v1/src/test/resources/logback-spring.xml new file mode 100644 index 0000000..33b2cef --- /dev/null +++ b/redaction-report-service-v1/redaction-report-service-server-v1/src/test/resources/logback-spring.xml @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + \ No newline at end of file