Marmelator/red-ui
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1.4k Wiki Activity

RED-5514 - CSV file encoding issues

Browse Source
This commit is contained in:
Timo Bejan 2022-11-21 12:10:18 +02:00
parent 9624115532
commit 4234ed52c9
2 changed files with 35 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
apps/red-ui/src/app/modules/admin/dialogs/file-attributes-csv-import-dialog/file-attributes-csv-import-dialog.component.html
View File

@ -34,6 +34,7 @@
<div class="iqser-input-group required w-110">
<label translate="file-attributes-csv-import.delimiter"></label>
<input
(keyup)="readFile(false)"
[placeholder]="'file-attributes-csv-import.delimiter-placeholder' | translate"
formControlName="delimiter"
name="delimiter"
@ -43,12 +44,12 @@
<div class="iqser-input-group required w-160">
<label translate="file-attributes-csv-import.encoding"></label>
<input
[placeholder]="'file-attributes-csv-import.encoding-placeholder' | translate"
formControlName="encoding"
name="encoding"
type="text"
/>
<mat-select formControlName="encoding" (selectionChange)="readFile()">
<mat-option *ngFor="let type of encodingTypeOptions" [value]="type">
{{ translations[type] | translate }}
</mat-option>
</mat-select>
</div>
<iqser-circle-button
(action)="readFile()"

32
apps/red-ui/src/app/modules/admin/dialogs/file-attributes-csv-import-dialog/file-attributes-csv-import-dialog.component.ts
View File

@ -6,8 +6,9 @@ import { firstValueFrom, Observable } from 'rxjs';
import { map, startWith } from 'rxjs/operators';
import { ListingComponent, listingProvidersFactory, TableColumnConfig, Toaster, trackByFactory } from '@iqser/common-ui';
import { marker as _ } from '@biesbjerg/ngx-translate-extract-marker';
import { FileAttributeConfig, FileAttributeConfigTypes, IField, IFileAttributesConfig } from '@red/domain';
import { FileAttributeConfig, FileAttributeConfigTypes, FileAttributeEncodingTypes, IField, IFileAttributesConfig } from '@red/domain';
import { FileAttributesService } from '@services/entity-services/file-attributes.service';
import { fileAttributeEncodingTypesTranslations } from '@translations/file-attribute-encoding-types-translations';
interface IFileAttributesCSVImportData {
readonly csv: File;
@ -33,6 +34,8 @@ export class FileAttributesCsvImportDialogComponent extends ListingComponent<IFi
keepPreview = false;
columnSample = [];
initialParseConfig: { delimiter?: string; encoding?: string } = {};
readonly encodingTypeOptions = Object.keys(FileAttributeEncodingTypes);
readonly translations = fileAttributeEncodingTypesTranslations;
readonly tableHeaderLabel = '';
readonly trackBy = trackByFactory();
@ -55,7 +58,7 @@ export class FileAttributesCsvImportDialogComponent extends ListingComponent<IFi
);
}
readFile() {
readFile(updateDelimiter: boolean = true) {
const reader = new FileReader();
reader.addEventListener('load', event => {
const parsedCsv = <any>event.target.result;
@ -63,7 +66,10 @@ export class FileAttributesCsvImportDialogComponent extends ListingComponent<IFi
header: true,
delimiter: this.form.get('delimiter').value,
});
if (updateDelimiter) {
this.form.patchValue({ delimiter: this.parseResult.meta.delimiter });
}
// Filter duplicate columns
if (this.parseResult?.data?.length) {
@ -108,7 +114,7 @@ export class FileAttributesCsvImportDialogComponent extends ListingComponent<IFi
encoding: this.form.get('encoding').value,
};
});
reader.readAsText(this.data.csv, this.form.get('encoding').value);
reader.readAsText(this.data.csv, this._mapEncoding(this.form.get('encoding').value));
}
getSample(csvColumn: string) {
@ -202,7 +208,7 @@ export class FileAttributesCsvImportDialogComponent extends ListingComponent<IFi
return this._formBuilder.group({
filenameMappingColumnHeaderName: ['', [Validators.required, this._autocompleteStringValidator()]],
delimiter: [undefined, Validators.required],
encoding: ['UTF-8', Validators.required],
encoding: [this.data.existingConfiguration?.encoding, Validators.required],
});
}
@ -219,7 +225,7 @@ export class FileAttributesCsvImportDialogComponent extends ListingComponent<IFi
const sample = this.getSample(csvColumn);
const isNumber = sample && !isNaN(sample);
return {
id: Buffer.from(csvColumn, 'base64').toString(),
id: btoa(unescape(encodeURIComponent(csvColumn))),
csvColumn,
name: csvColumn,
searchKey: csvColumn,
@ -228,4 +234,20 @@ export class FileAttributesCsvImportDialogComponent extends ListingComponent<IFi
primaryAttribute: false,
};
}
private _mapEncoding(encoding: string) {
if (!encoding) {
return 'UTF-8';
}
if (encoding.toLowerCase() === 'utf-8' || encoding.toLowerCase() === 'utf_8') {
return 'UTF-8';
}
if (encoding.toLowerCase() === 'ascii') {
return 'ascii';
}
if (encoding.toLowerCase().startsWith('iso')) {
return 'ISO-8859-1';
}
return 'UTF-8';
}
}