From 874fdc85ee136db63a2797c109bb366aa8aaf335 Mon Sep 17 00:00:00 2001 From: Andrei Isvoran Date: Mon, 9 Sep 2024 16:10:06 +0300 Subject: [PATCH] RED-9541 - Add extra CSV validation --- .../FileAttributesManagementService.java | 37 +++++++++-- .../integration/tests/FileAttributeTest.java | 63 +++++++++++++++++++ .../csv/fileattributes_missing_comma.csv | 2 + .../fileattributes_missing_quotation_mark.csv | 2 + 4 files changed, 99 insertions(+), 5 deletions(-) create mode 100644 persistence-service-v1/persistence-service-server-v1/src/test/resources/files/csv/fileattributes_missing_comma.csv create mode 100644 persistence-service-v1/persistence-service-server-v1/src/test/resources/files/csv/fileattributes_missing_quotation_mark.csv diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileAttributesManagementService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileAttributesManagementService.java index 16b0e3e50..75097035e 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileAttributesManagementService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileAttributesManagementService.java @@ -43,7 +43,6 @@ import com.opencsv.CSVParserBuilder; import com.opencsv.CSVReader; import com.opencsv.CSVReaderBuilder; import com.opencsv.exceptions.CsvException; -import com.opencsv.exceptions.CsvValidationException; import jakarta.transaction.Transactional; import lombok.RequiredArgsConstructor; @@ -139,7 +138,7 @@ public class FileAttributesManagementService { @SuppressWarnings("checkstyle:all") - private void validateCsv(byte[] csvFileBytes, String delimiter, String encoding) { + public void validateCsv(byte[] csvFileBytes, String delimiter, String encoding) { if (delimiter.length() != 1) { throw new IllegalArgumentException("Delimiter must be a single character."); @@ -150,16 +149,44 @@ public class FileAttributesManagementService { try (CSVReader csvReader = new CSVReaderBuilder(new InputStreamReader(new ByteArrayInputStream(csvFileBytes), charset)).withCSVParser(new CSVParserBuilder().withSeparator( delimiterChar).build()).build()) { - while (csvReader.readNext() != null) { - // Intentionally empty: reading lines for validation purposes + String[] nextLine; + int lineNumber = 0; + int expectedColumnCount = -1; + + while ((nextLine = csvReader.readNext()) != null) { + lineNumber++; + + if (expectedColumnCount == -1) { + expectedColumnCount = nextLine.length; + } else if (nextLine.length != expectedColumnCount) { + throw new BadRequestException("Invalid CSV file format at line " + lineNumber + ": Expected " + expectedColumnCount + " columns but found " + nextLine.length); + } + + for (String field : nextLine) { + if (field != null && !field.isEmpty()) { + validateQuotesInField(field, lineNumber); + } + } } - } catch (IOException | CsvValidationException e) { + } catch (Exception e) { throw new BadRequestException("Invalid CSV file format: " + e.getMessage(), e); } } + private void validateQuotesInField(String field, int lineNumber) { + + boolean startsWithQuote = field.startsWith("\""); + boolean endsWithQuote = field.endsWith("\""); + + if (startsWithQuote && !endsWithQuote) { + throw new BadRequestException("Invalid CSV format at line " + lineNumber + ": Unmatched quotation marks. in field '" + field + "'"); + } + + } + + @SuppressFBWarnings("RCN_REDUNDANT_NULLCHECK_WOULD_HAVE_BEEN_A_NPE") private List> getCsvRecords(byte[] csv, String delimiter, String encoding) { diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileAttributeTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileAttributeTest.java index 481c9368a..235c50013 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileAttributeTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileAttributeTest.java @@ -27,6 +27,7 @@ import com.iqser.red.service.peristence.v1.server.integration.client.UploadClien import com.iqser.red.service.peristence.v1.server.integration.service.DossierTesterAndProvider; import com.iqser.red.service.peristence.v1.server.integration.service.FileTesterAndProvider; import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest; +import com.iqser.red.service.persistence.management.v1.processor.service.FileAttributesManagementService; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttributes; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttributesConfig; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier; @@ -58,6 +59,9 @@ public class FileAttributeTest extends AbstractPersistenceServerServiceTest { @Autowired private FileManagementClient fileManagementClient; + @Autowired + private FileAttributesManagementService fileAttribute; + @SneakyThrows @Test @@ -311,4 +315,63 @@ public class FileAttributeTest extends AbstractPersistenceServerServiceTest { assertEquals(400, exception.status()); } + + @Test + @SneakyThrows + public void testUploadCSV() { + + var dossier = dossierTesterAndProvider.provideTestDossier(); + + var generalConfig = new FileAttributesConfig(); + generalConfig.setDelimiter(","); + generalConfig.setEncoding("UTF-8"); + generalConfig.setFilenameMappingColumnHeaderName("Path"); + + fileAttributeConfigClient.setFileAttributesConfig(dossier.getDossierTemplateId(), generalConfig); + + FileAttributeConfig vertebrateStudy = new FileAttributeConfig(); + vertebrateStudy.setPrimaryAttribute(false); + vertebrateStudy.setLabel("Vertebrate Study"); + vertebrateStudy.setCsvColumnHeader("Vertebrate Study"); + vertebrateStudy.setType(FileAttributeType.TEXT); + vertebrateStudy.setFilterable(true); + vertebrateStudy.setDisplayedInFileList(true); + fileAttributeConfigClient.addOrUpdateFileAttribute(dossier.getDossierTemplateId(), vertebrateStudy); + + FileAttributeConfig minorVersion = new FileAttributeConfig(); + minorVersion.setPrimaryAttribute(false); + minorVersion.setLabel("Minor Version Number"); + minorVersion.setCsvColumnHeader("Minor Version Number"); + minorVersion.setType(FileAttributeType.DATE); + minorVersion.setFilterable(true); + minorVersion.setDisplayedInFileList(true); + fileAttributeConfigClient.addOrUpdateFileAttribute(dossier.getDossierTemplateId(), minorVersion); + + FileAttributeConfig majorVersion = new FileAttributeConfig(); + majorVersion.setPrimaryAttribute(false); + majorVersion.setLabel("Major Version Number"); + majorVersion.setCsvColumnHeader("Major Version Number"); + majorVersion.setType(FileAttributeType.NUMBER); + majorVersion.setFilterable(true); + majorVersion.setDisplayedInFileList(true); + fileAttributeConfigClient.addOrUpdateFileAttribute(dossier.getDossierTemplateId(), majorVersion); + + var missingComma = new MockMultipartFile("file.csv", + "fileattributes_missing_comma.csv", + "application/csv", + IOUtils.toByteArray(new ClassPathResource("files/csv/fileattributes_missing_comma.csv").getInputStream())); + + var result = assertThrows(FeignException.class, () -> uploadClient.upload(missingComma, dossier.getId(), false, false)); + assertTrue(result.getMessage().contains("Invalid CSV file format: Invalid CSV file format at line 2: Expected 5 columns but found 4")); + + + var missingQuotation = new MockMultipartFile("file.csv", + "fileattributes_missing_quotation_mark.csv", + "application/csv", + IOUtils.toByteArray(new ClassPathResource("files/csv/fileattributes_missing_quotation_mark.csv").getInputStream())); + + result = assertThrows(FeignException.class, () -> uploadClient.upload(missingQuotation, dossier.getId(), false, false)); + assertTrue(result.getMessage().contains("Invalid CSV file format: Unterminated quoted field at end of CSV line. Beginning of lost text: [4.636.0,4.363.0,4.363.0\\n]")); + } + } diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/resources/files/csv/fileattributes_missing_comma.csv b/persistence-service-v1/persistence-service-server-v1/src/test/resources/files/csv/fileattributes_missing_comma.csv new file mode 100644 index 000000000..4360340e2 --- /dev/null +++ b/persistence-service-v1/persistence-service-server-v1/src/test/resources/files/csv/fileattributes_missing_comma.csv @@ -0,0 +1,2 @@ +Path,"Document Title","Major Version Number","Minor Version Number","Vertebrate Study" +"402Study.pdf","My Title","4.636.0","4.363.0""4.363.0" \ No newline at end of file diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/resources/files/csv/fileattributes_missing_quotation_mark.csv b/persistence-service-v1/persistence-service-server-v1/src/test/resources/files/csv/fileattributes_missing_quotation_mark.csv new file mode 100644 index 000000000..012aea358 --- /dev/null +++ b/persistence-service-v1/persistence-service-server-v1/src/test/resources/files/csv/fileattributes_missing_quotation_mark.csv @@ -0,0 +1,2 @@ +Path,"Document Title","Major Version Number","Minor Version Number","Vertebrate Study" +"402Study.pdf","My Title","4.636.0,"4.363.0","4.363.0" \ No newline at end of file