From e5ea667ea108294c547daa58d7fdba4c6cebb1d0 Mon Sep 17 00:00:00 2001 From: corinaolariu Date: Fri, 22 Nov 2024 11:32:18 +0200 Subject: [PATCH] RED-10443 - 500 Error occurs when selecting ISO-8859-1 as Encoding Type for any CSV File Format - accept only the IS-8859-1 as encoding. Meaningful message (400) is returned in case of bad encoding - update unit tests and add unit test --- .../resource/FileAttributesResource.java | 6 +++- .../service/ComponentMappingService.java | 17 ++-------- .../FileAttributesManagementService.java | 11 +++--- ...FileAttributeConfigPersistenceService.java | 6 ++-- .../processor/utils/StringEncodingUtils.java | 18 ++++++++++ .../tests/ComponentMappingTest.java | 7 ++-- .../integration/tests/FileAttributeTest.java | 34 +++++++++++++++---- .../v1/server/integration/tests/FileTest.java | 3 +- 8 files changed, 67 insertions(+), 35 deletions(-) diff --git a/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/FileAttributesResource.java b/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/FileAttributesResource.java index 9508c4b4d..b501e8ffc 100644 --- a/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/FileAttributesResource.java +++ b/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/FileAttributesResource.java @@ -44,7 +44,11 @@ public interface FileAttributesResource { String FILE_ID = "fileId"; String FILE_ID_PATH_VARIABLE = "/{" + FILE_ID + "}"; - Set encodingList = Sets.newHashSet("ISO", "ASCII", "UTF-8"); + String UTF_ENCODING = "UTF-8"; + String ASCII_ENCODING = "ASCII"; + String ISO_ENCODING = "ISO-8859-1"; + + Set encodingList = Sets.newHashSet(ISO_ENCODING, ASCII_ENCODING, UTF_ENCODING); @ResponseBody diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ComponentMappingService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ComponentMappingService.java index 0cdafc753..cf5ec6a05 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ComponentMappingService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ComponentMappingService.java @@ -24,6 +24,7 @@ import com.iqser.red.service.persistence.management.v1.processor.mapper.Componen import com.iqser.red.service.persistence.management.v1.processor.model.ComponentMapping; import com.iqser.red.service.persistence.management.v1.processor.model.ComponentMappingDownloadModel; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.utils.StringEncodingUtils; import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata; import com.opencsv.CSVParserBuilder; import com.opencsv.CSVReader; @@ -107,7 +108,7 @@ public class ComponentMappingService { String fileName, char quoteChar) { - Charset charset = resolveCharset(encoding); + Charset charset = StringEncodingUtils.resolveCharset(encoding); CsvStats stats = sortCSVFile(delimiter, mappingFile, charset, quoteChar); @@ -126,20 +127,6 @@ public class ComponentMappingService { } - private static Charset resolveCharset(String encoding) { - - try { - return Charset.forName(encoding); - } catch (IllegalCharsetNameException e) { - throw new BadRequestException("Invalid character encoding: " + encoding); - } catch (UnsupportedCharsetException e) { - throw new BadRequestException("Unsupported character encoding: " + encoding); - } catch (IllegalArgumentException e) { - throw new BadRequestException("Encoding can't be null."); - } - } - - private static CsvStats sortCSVFile(char delimiter, File mappingFile, Charset charset, char quoteChar) throws BadRequestException, IOException { Path tempFile = Files.createTempFile("mapping", ".tmp"); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileAttributesManagementService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileAttributesManagementService.java index 75097035e..d899dc85b 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileAttributesManagementService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileAttributesManagementService.java @@ -1,5 +1,7 @@ package com.iqser.red.service.persistence.management.v1.processor.service; +import static com.iqser.red.service.persistence.service.v1.api.external.resource.FileAttributesResource.ASCII_ENCODING; +import static com.iqser.red.service.persistence.service.v1.api.external.resource.FileAttributesResource.ISO_ENCODING; import static com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileAttributeTypeFormats.FILE_ATTRIBUTE_TYPE_DATE_FORMAT; import static com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileAttributeTypeFormats.FILE_ATTRIBUTE_TYPE_NUMBER_REGEX; @@ -32,6 +34,7 @@ import com.iqser.red.service.persistence.management.v1.processor.exception.Confl import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileAttributeConfigPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.utils.StringEncodingUtils; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.ImportCsvRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.ImportCsvResponse; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileAttributeType; @@ -59,10 +62,6 @@ public class FileAttributesManagementService { private final DossierPersistenceService dossierPersistenceService; private final IndexingService indexingService; - public static String UTF_ENCODING = "UTF-8"; - public static String ASCII_ENCODING = "ASCII"; - public static String ISO_ENCODING = "ISO"; - @Transactional public ImportCsvResponse importCsv(String dossierId, ImportCsvRequest importCsvRequest) { @@ -144,7 +143,7 @@ public class FileAttributesManagementService { throw new IllegalArgumentException("Delimiter must be a single character."); } char delimiterChar = delimiter.charAt(0); - Charset charset = Charset.forName(encoding); + Charset charset = StringEncodingUtils.resolveCharset(encoding); try (CSVReader csvReader = new CSVReaderBuilder(new InputStreamReader(new ByteArrayInputStream(csvFileBytes), charset)).withCSVParser(new CSVParserBuilder().withSeparator( delimiterChar).build()).build()) { @@ -214,7 +213,7 @@ public class FileAttributesManagementService { if (ASCII_ENCODING.equalsIgnoreCase(encoding) || StandardCharsets.US_ASCII.name().equalsIgnoreCase(encoding)) { return StandardCharsets.US_ASCII; } - // accept both "ISO" (non-unique name) and the actual name "US-ASCII" of the charset + // accept only name "ISO_8859_1" of the charset if (ISO_ENCODING.equalsIgnoreCase(encoding) || StandardCharsets.ISO_8859_1.name().equalsIgnoreCase(encoding)) { return StandardCharsets.ISO_8859_1; } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileAttributeConfigPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileAttributeConfigPersistenceService.java index 739679c89..60bf63dd4 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileAttributeConfigPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileAttributeConfigPersistenceService.java @@ -1,8 +1,8 @@ package com.iqser.red.service.persistence.management.v1.processor.service.persistence; -import static com.iqser.red.service.persistence.management.v1.processor.service.FileAttributesManagementService.ASCII_ENCODING; -import static com.iqser.red.service.persistence.management.v1.processor.service.FileAttributesManagementService.ISO_ENCODING; -import static com.iqser.red.service.persistence.management.v1.processor.service.FileAttributesManagementService.UTF_ENCODING; +import static com.iqser.red.service.persistence.service.v1.api.external.resource.FileAttributesResource.ASCII_ENCODING; +import static com.iqser.red.service.persistence.service.v1.api.external.resource.FileAttributesResource.ISO_ENCODING; +import static com.iqser.red.service.persistence.service.v1.api.external.resource.FileAttributesResource.UTF_ENCODING; import java.util.List; import java.util.Objects; diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/StringEncodingUtils.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/StringEncodingUtils.java index ea215d0b1..c8f1ad1a3 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/StringEncodingUtils.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/StringEncodingUtils.java @@ -1,10 +1,15 @@ package com.iqser.red.service.persistence.management.v1.processor.utils; import java.net.URLEncoder; +import java.nio.charset.Charset; +import java.nio.charset.IllegalCharsetNameException; import java.nio.charset.StandardCharsets; +import java.nio.charset.UnsupportedCharsetException; import org.apache.commons.lang3.StringUtils; +import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException; + import lombok.experimental.UtilityClass; @UtilityClass @@ -30,4 +35,17 @@ public final class StringEncodingUtils { return result.toString(); } + public static Charset resolveCharset(String encoding) { + + try { + return Charset.forName(encoding); + } catch (IllegalCharsetNameException e) { + throw new BadRequestException("Invalid character encoding: " + encoding); + } catch (UnsupportedCharsetException e) { + throw new BadRequestException("Unsupported character encoding: " + encoding); + } catch (IllegalArgumentException e) { + throw new BadRequestException("Encoding can't be null."); + } + } + } diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ComponentMappingTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ComponentMappingTest.java index f5912c802..4daa1980f 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ComponentMappingTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ComponentMappingTest.java @@ -1,5 +1,6 @@ package com.iqser.red.service.peristence.v1.server.integration.tests; +import static com.iqser.red.service.persistence.service.v1.api.external.resource.FileAttributesResource.UTF_ENCODING; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -56,7 +57,7 @@ public class ComponentMappingTest extends AbstractPersistenceServerServiceTest { ComponentMappingMetadataModel componentMappingMetadataModel = dossierTemplateExternalClient.uploadMapping(dossierTemplate.getId(), mockMultipartFile, "file", - "UTF-8", + UTF_ENCODING, ",", "\""); @@ -81,7 +82,7 @@ public class ComponentMappingTest extends AbstractPersistenceServerServiceTest { componentMappingMetadataModel.getId(), updateMockMultipartFile, "file", - "UTF-8", + UTF_ENCODING, ",", "\""); @@ -101,7 +102,7 @@ public class ComponentMappingTest extends AbstractPersistenceServerServiceTest { IOUtils.toByteArray(new ClassPathResource("files/componentmapping/empty.csv").getInputStream())); String id = dossierTemplate.getId(); - var result = assertThrows(FeignException.class, () -> dossierTemplateExternalClient.uploadMapping(id, mockMultipartFile, "file", "UTF-8", ",", "\"")); + var result = assertThrows(FeignException.class, () -> dossierTemplateExternalClient.uploadMapping(id, mockMultipartFile, "file", UTF_ENCODING, ",", "\"")); assertTrue(result.getMessage().contains("CSV file can not be empty!")); } diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileAttributeTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileAttributeTest.java index 93ba08588..db3478d84 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileAttributeTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileAttributeTest.java @@ -1,7 +1,8 @@ package com.iqser.red.service.peristence.v1.server.integration.tests; -import static com.iqser.red.service.persistence.management.v1.processor.service.FileAttributesManagementService.ASCII_ENCODING; -import static com.iqser.red.service.persistence.management.v1.processor.service.FileAttributesManagementService.UTF_ENCODING; +import static com.iqser.red.service.persistence.service.v1.api.external.resource.FileAttributesResource.ASCII_ENCODING; +import static com.iqser.red.service.persistence.service.v1.api.external.resource.FileAttributesResource.ISO_ENCODING; +import static com.iqser.red.service.persistence.service.v1.api.external.resource.FileAttributesResource.UTF_ENCODING; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -73,7 +74,7 @@ public class FileAttributeTest extends AbstractPersistenceServerServiceTest { var generalConfig = new FileAttributesConfig(); generalConfig.setDelimiter(","); - generalConfig.setEncoding("UTF-8"); + generalConfig.setEncoding(UTF_ENCODING); generalConfig.setFilenameMappingColumnHeaderName("Name"); fileAttributeConfigClient.setFileAttributesConfig(dossier.getDossierTemplateId(), generalConfig); @@ -196,7 +197,7 @@ public class FileAttributeTest extends AbstractPersistenceServerServiceTest { var generalConfig = new FileAttributesConfig(); generalConfig.setDelimiter(","); - generalConfig.setEncoding("UTF-8"); + generalConfig.setEncoding(UTF_ENCODING); generalConfig.setFilenameMappingColumnHeaderName("Name"); fileAttributeConfigClient.setFileAttributesConfig(dossier.getDossierTemplateId(), generalConfig); @@ -248,7 +249,7 @@ public class FileAttributeTest extends AbstractPersistenceServerServiceTest { var generalConfig = new FileAttributesConfig(); generalConfig.setDelimiter(","); - generalConfig.setEncoding("UTF-8"); + generalConfig.setEncoding(UTF_ENCODING); generalConfig.setFilenameMappingColumnHeaderName("Name"); fileAttributeConfigClient.setFileAttributesConfig(dossier.getDossierTemplateId(), generalConfig); @@ -324,7 +325,7 @@ public class FileAttributeTest extends AbstractPersistenceServerServiceTest { var generalConfig = new FileAttributesConfig(); generalConfig.setDelimiter(","); - generalConfig.setEncoding("UTF-8"); + generalConfig.setEncoding(UTF_ENCODING); generalConfig.setFilenameMappingColumnHeaderName("Path"); fileAttributeConfigClient.setFileAttributesConfig(dossier.getDossierTemplateId(), generalConfig); @@ -374,4 +375,25 @@ public class FileAttributeTest extends AbstractPersistenceServerServiceTest { assertTrue(result.getMessage().contains("Invalid CSV file format: Unterminated quoted field at end of CSV line. Beginning of lost text: [4.636.0,4.363.0,4.363.0\\n]") || result.getMessage().contains("Invalid CSV file format: Unterminiertes Anführungszeichen am Ende einer CSV-Zeile. Anfang des verlorenen Textes: [4.636.0,4.363.0,4.363.0\\n]")); } + @Test + public void testParsingEncoding() { + var dossier = dossierTesterAndProvider.provideTestDossier(); + + + var generalConfig = new FileAttributesConfig(); + generalConfig.setDelimiter(","); + generalConfig.setEncoding("ISO"); + generalConfig.setFilenameMappingColumnHeaderName("Name"); + + var e = assertThrows(FeignException.class, + () -> fileAttributeConfigClient.setFileAttributesConfig(dossier.getDossierTemplateId(), generalConfig)); + assertEquals(400, e.status()); + generalConfig.setEncoding(ISO_ENCODING); + fileAttributeConfigClient.setFileAttributesConfig(dossier.getDossierTemplateId(), generalConfig); + var loadedConfig = fileAttributeConfigClient.getFileAttributesConfiguration(dossier.getDossierTemplateId()); + assertThat(loadedConfig.getEncoding()).isEqualTo(ISO_ENCODING); + + + } + } diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java index 3bee17a28..8ca1a6add 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java @@ -1,5 +1,6 @@ package com.iqser.red.service.peristence.v1.server.integration.tests; +import static com.iqser.red.service.persistence.service.v1.api.external.resource.FileAttributesResource.UTF_ENCODING; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -765,7 +766,7 @@ public class FileTest extends AbstractPersistenceServerServiceTest { fileManagementStorageService.storeObject(dossier.getId(), fileId, FileType.UNTOUCHED, new ByteArrayInputStream("test".getBytes(StandardCharsets.UTF_8))); when(fileAttributeConfigPersistenceService.getFileAttributesGeneralConfiguration(anyString())).thenReturn(FileAttributesGeneralConfigurationEntity.builder() .delimiter(",") - .encoding("UTF-8") + .encoding(UTF_ENCODING) .build()); when(fileAttributeConfigPersistenceService.getFileAttributes(anyString())).thenReturn(Collections.emptyList()); assertThrows(FeignException.class, () -> uploadClient.upload(malformedCsvFile, dossier.getId(), false, false));