Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

RED-9541 - Add extra CSV validation

Browse Source
This commit is contained in:
Andrei Isvoran 2024-09-09 16:10:06 +03:00
parent d144b31747
commit 874fdc85ee
4 changed files with 99 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileAttributesManagementService.java
View File

@ -43,7 +43,6 @@ import com.opencsv.CSVParserBuilder;
import com.opencsv.CSVReader;
import com.opencsv.CSVReaderBuilder;
import com.opencsv.exceptions.CsvException;
import com.opencsv.exceptions.CsvValidationException;
import jakarta.transaction.Transactional;
import lombok.RequiredArgsConstructor;
@ -139,7 +138,7 @@ public class FileAttributesManagementService {
@SuppressWarnings("checkstyle:all")
private void validateCsv(byte[] csvFileBytes, String delimiter, String encoding) {
public void validateCsv(byte[] csvFileBytes, String delimiter, String encoding) {
if (delimiter.length() != 1) {
throw new IllegalArgumentException("Delimiter must be a single character.");
@ -150,16 +149,44 @@ public class FileAttributesManagementService {
try (CSVReader csvReader = new CSVReaderBuilder(new InputStreamReader(new ByteArrayInputStream(csvFileBytes), charset)).withCSVParser(new CSVParserBuilder().withSeparator(
delimiterChar).build()).build()) {
while (csvReader.readNext() != null) {
// Intentionally empty: reading lines for validation purposes
String[] nextLine;
int lineNumber = 0;
int expectedColumnCount = -1;
while ((nextLine = csvReader.readNext()) != null) {
lineNumber++;
if (expectedColumnCount == -1) {
expectedColumnCount = nextLine.length;
} else if (nextLine.length != expectedColumnCount) {
throw new BadRequestException("Invalid CSV file format at line " + lineNumber + ": Expected " + expectedColumnCount + " columns but found " + nextLine.length);
}
for (String field : nextLine) {
if (field != null && !field.isEmpty()) {
validateQuotesInField(field, lineNumber);
}
}
}
} catch (IOException | CsvValidationException e) {
} catch (Exception e) {
throw new BadRequestException("Invalid CSV file format: " + e.getMessage(), e);
}
}
private void validateQuotesInField(String field, int lineNumber) {
boolean startsWithQuote = field.startsWith("\"");
boolean endsWithQuote = field.endsWith("\"");
if (startsWithQuote && !endsWithQuote) {
throw new BadRequestException("Invalid CSV format at line " + lineNumber + ": Unmatched quotation marks. in field '" + field + "'");
}
}
@SuppressFBWarnings("RCN_REDUNDANT_NULLCHECK_WOULD_HAVE_BEEN_A_NPE")
private List<List<String>> getCsvRecords(byte[] csv, String delimiter, String encoding) {

63
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileAttributeTest.java
View File

@ -27,6 +27,7 @@ import com.iqser.red.service.peristence.v1.server.integration.client.UploadClien
import com.iqser.red.service.peristence.v1.server.integration.service.DossierTesterAndProvider;
import com.iqser.red.service.peristence.v1.server.integration.service.FileTesterAndProvider;
import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest;
import com.iqser.red.service.persistence.management.v1.processor.service.FileAttributesManagementService;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttributes;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttributesConfig;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier;
@ -58,6 +59,9 @@ public class FileAttributeTest extends AbstractPersistenceServerServiceTest {
@Autowired
private FileManagementClient fileManagementClient;
@Autowired
private FileAttributesManagementService fileAttribute;
@SneakyThrows
@Test
@ -311,4 +315,63 @@ public class FileAttributeTest extends AbstractPersistenceServerServiceTest {
assertEquals(400, exception.status());
}
@Test
@SneakyThrows
public void testUploadCSV() {
var dossier = dossierTesterAndProvider.provideTestDossier();
var generalConfig = new FileAttributesConfig();
generalConfig.setDelimiter(",");
generalConfig.setEncoding("UTF-8");
generalConfig.setFilenameMappingColumnHeaderName("Path");
fileAttributeConfigClient.setFileAttributesConfig(dossier.getDossierTemplateId(), generalConfig);
FileAttributeConfig vertebrateStudy = new FileAttributeConfig();
vertebrateStudy.setPrimaryAttribute(false);
vertebrateStudy.setLabel("Vertebrate Study");
vertebrateStudy.setCsvColumnHeader("Vertebrate Study");
vertebrateStudy.setType(FileAttributeType.TEXT);
vertebrateStudy.setFilterable(true);
vertebrateStudy.setDisplayedInFileList(true);
fileAttributeConfigClient.addOrUpdateFileAttribute(dossier.getDossierTemplateId(), vertebrateStudy);
FileAttributeConfig minorVersion = new FileAttributeConfig();
minorVersion.setPrimaryAttribute(false);
minorVersion.setLabel("Minor Version Number");
minorVersion.setCsvColumnHeader("Minor Version Number");
minorVersion.setType(FileAttributeType.DATE);
minorVersion.setFilterable(true);
minorVersion.setDisplayedInFileList(true);
fileAttributeConfigClient.addOrUpdateFileAttribute(dossier.getDossierTemplateId(), minorVersion);
FileAttributeConfig majorVersion = new FileAttributeConfig();
majorVersion.setPrimaryAttribute(false);
majorVersion.setLabel("Major Version Number");
majorVersion.setCsvColumnHeader("Major Version Number");
majorVersion.setType(FileAttributeType.NUMBER);
majorVersion.setFilterable(true);
majorVersion.setDisplayedInFileList(true);
fileAttributeConfigClient.addOrUpdateFileAttribute(dossier.getDossierTemplateId(), majorVersion);
var missingComma = new MockMultipartFile("file.csv",
"fileattributes_missing_comma.csv",
"application/csv",
IOUtils.toByteArray(new ClassPathResource("files/csv/fileattributes_missing_comma.csv").getInputStream()));
var result = assertThrows(FeignException.class, () -> uploadClient.upload(missingComma, dossier.getId(), false, false));
assertTrue(result.getMessage().contains("Invalid CSV file format: Invalid CSV file format at line 2: Expected 5 columns but found 4"));
var missingQuotation = new MockMultipartFile("file.csv",
"fileattributes_missing_quotation_mark.csv",
"application/csv",
IOUtils.toByteArray(new ClassPathResource("files/csv/fileattributes_missing_quotation_mark.csv").getInputStream()));
result = assertThrows(FeignException.class, () -> uploadClient.upload(missingQuotation, dossier.getId(), false, false));
assertTrue(result.getMessage().contains("Invalid CSV file format: Unterminated quoted field at end of CSV line. Beginning of lost text: [4.636.0,4.363.0,4.363.0\\n]"));
}
}

2
persistence-service-v1/persistence-service-server-v1/src/test/resources/files/csv/fileattributes_missing_comma.csv Normal file
View File

@ -0,0 +1,2 @@
Path,"Document Title","Major Version Number","Minor Version Number","Vertebrate Study"
"402Study.pdf","My Title","4.636.0","4.363.0""4.363.0"
Can't render this file because it has a wrong number of fields in line 2.

2
persistence-service-v1/persistence-service-server-v1/src/test/resources/files/csv/fileattributes_missing_quotation_mark.csv Normal file
View File

@ -0,0 +1,2 @@
Path,"Document Title","Major Version Number","Minor Version Number","Vertebrate Study"
"402Study.pdf","My Title","4.636.0,"4.363.0","4.363.0"
Can't render this file because it contains an unexpected character in line 2 and column 36.