RED-5514 - CSV file encoding issues

apps/red-ui/src/app/modules/admin/dialogs/file-attributes-csv-import-dialog/file-attributes-csv-import-dialog.component.html

@@ -34,6 +34,7 @@
                     <div class="iqser-input-group required w-110">
                         <label translate="file-attributes-csv-import.delimiter"></label>
                         <input
+                            (keyup)="readFile(false)"
                             [placeholder]="'file-attributes-csv-import.delimiter-placeholder' | translate"
                             formControlName="delimiter"
                             name="delimiter"
@@ -43,12 +44,12 @@
 
                     <div class="iqser-input-group required w-160">
                         <label translate="file-attributes-csv-import.encoding"></label>
-                        <input
+
-                            [placeholder]="'file-attributes-csv-import.encoding-placeholder' | translate"
+                        <mat-select formControlName="encoding" (selectionChange)="readFile()">
-                            formControlName="encoding"
+                            <mat-option *ngFor="let type of encodingTypeOptions" [value]="type">
-                            name="encoding"
+                                {{ translations[type] | translate }}
-                            type="text"
+                            </mat-option>
-                        />
+                        </mat-select>
                     </div>
                     <iqser-circle-button
                         (action)="readFile()"

apps/red-ui/src/app/modules/admin/dialogs/file-attributes-csv-import-dialog/file-attributes-csv-import-dialog.component.ts

@@ -6,8 +6,9 @@ import { firstValueFrom, Observable } from 'rxjs';
 import { map, startWith } from 'rxjs/operators';
 import { ListingComponent, listingProvidersFactory, TableColumnConfig, Toaster, trackByFactory } from '@iqser/common-ui';
 import { marker as _ } from '@biesbjerg/ngx-translate-extract-marker';
-import { FileAttributeConfig, FileAttributeConfigTypes, IField, IFileAttributesConfig } from '@red/domain';
+import { FileAttributeConfig, FileAttributeConfigTypes, FileAttributeEncodingTypes, IField, IFileAttributesConfig } from '@red/domain';
 import { FileAttributesService } from '@services/entity-services/file-attributes.service';
+import { fileAttributeEncodingTypesTranslations } from '@translations/file-attribute-encoding-types-translations';
 
 interface IFileAttributesCSVImportData {
     readonly csv: File;
@@ -33,6 +34,8 @@ export class FileAttributesCsvImportDialogComponent extends ListingComponent<IFi
     keepPreview = false;
     columnSample = [];
     initialParseConfig: { delimiter?: string; encoding?: string } = {};
+    readonly encodingTypeOptions = Object.keys(FileAttributeEncodingTypes);
+    readonly translations = fileAttributeEncodingTypesTranslations;
     readonly tableHeaderLabel = '';
     readonly trackBy = trackByFactory();
 
@@ -55,7 +58,7 @@ export class FileAttributesCsvImportDialogComponent extends ListingComponent<IFi
         );
     }
 
-    readFile() {
+    readFile(updateDelimiter: boolean = true) {
         const reader = new FileReader();
         reader.addEventListener('load', event => {
             const parsedCsv = <any>event.target.result;
@@ -63,7 +66,10 @@ export class FileAttributesCsvImportDialogComponent extends ListingComponent<IFi
                 header: true,
                 delimiter: this.form.get('delimiter').value,
             });
+
+            if (updateDelimiter) {
                 this.form.patchValue({ delimiter: this.parseResult.meta.delimiter });
+            }
 
             // Filter duplicate columns
             if (this.parseResult?.data?.length) {
@@ -108,7 +114,7 @@ export class FileAttributesCsvImportDialogComponent extends ListingComponent<IFi
                 encoding: this.form.get('encoding').value,
             };
         });
-        reader.readAsText(this.data.csv, this.form.get('encoding').value);
+        reader.readAsText(this.data.csv, this._mapEncoding(this.form.get('encoding').value));
     }
 
     getSample(csvColumn: string) {
@@ -202,7 +208,7 @@ export class FileAttributesCsvImportDialogComponent extends ListingComponent<IFi
         return this._formBuilder.group({
             filenameMappingColumnHeaderName: ['', [Validators.required, this._autocompleteStringValidator()]],
             delimiter: [undefined, Validators.required],
-            encoding: ['UTF-8', Validators.required],
+            encoding: [this.data.existingConfiguration?.encoding, Validators.required],
         });
     }
 
@@ -219,7 +225,7 @@ export class FileAttributesCsvImportDialogComponent extends ListingComponent<IFi
         const sample = this.getSample(csvColumn);
         const isNumber = sample && !isNaN(sample);
         return {
-            id: Buffer.from(csvColumn, 'base64').toString(),
+            id: btoa(unescape(encodeURIComponent(csvColumn))),
             csvColumn,
             name: csvColumn,
             searchKey: csvColumn,
@@ -228,4 +234,20 @@ export class FileAttributesCsvImportDialogComponent extends ListingComponent<IFi
             primaryAttribute: false,
         };
     }
+
+    private _mapEncoding(encoding: string) {
+        if (!encoding) {
+            return 'UTF-8';
+        }
+        if (encoding.toLowerCase() === 'utf-8' || encoding.toLowerCase() === 'utf_8') {
+            return 'UTF-8';
+        }
+        if (encoding.toLowerCase() === 'ascii') {
+            return 'ascii';
+        }
+        if (encoding.toLowerCase().startsWith('iso')) {
+            return 'ISO-8859-1';
+        }
+        return 'UTF-8';
+    }
 }