RED-2208: Smartcat WIP

tools/auto-i18n/i18n-cleanup.ts

@@ -1,147 +0,0 @@
-import * as fs from 'fs';
-import axios from 'axios';
-// @ts-ignore
-import * as xliff from 'xliff';
-
-// import { TranslateMessageFormatCompiler } from 'ngx-translate-messageformat-compiler';
-
-function flatten(data: any) {
-    const result: any = {};
-
-    function recurse(cur: any, prop: any) {
-        if (Object(cur) !== cur) {
-            result[prop] = cur;
-        } else if (Array.isArray(cur)) {
-            let l = 0;
-            for (let i = 0, l = cur.length; i < l; i++) recurse(cur[i], prop + '[' + i + ']');
-            if (l === 0) result[prop] = [];
-        } else {
-            let isEmpty = true;
-            for (const p in cur) {
-                isEmpty = false;
-                recurse(cur[p], prop ? prop + '.' + p : p);
-            }
-            if (isEmpty && prop) result[prop] = {};
-        }
-    }
-
-    recurse(data, '');
-    return result;
-}
-
-function unflatten(data: any) {
-    if (Object(data) !== data || Array.isArray(data)) return data;
-    const regex = /\.?([^.\[\]]+)|\[(\d+)\]/g,
-        resultholder: any = {};
-    for (const p in data) {
-        let cur: any = resultholder,
-            prop = '',
-            m: any;
-        while ((m = regex.exec(p))) {
-            cur = cur[prop] || (cur[prop] = m[2] ? [] : {});
-            prop = m[2] || m[1];
-        }
-        cur[prop] = data[p];
-    }
-    return resultholder[''] || resultholder;
-}
-
-async function execute() {
-    // const flatGerman = JSON.parse(fs.readFileSync('de-flat.json', 'utf-8'));
-
-    const german = JSON.parse(fs.readFileSync('./../../apps/red-ui/src/assets/i18n/de.json', 'utf-8'));
-    const flatGerman = flatten(german);
-
-    const english = JSON.parse(fs.readFileSync('./../../apps/red-ui/src/assets/i18n/en.json', 'utf-8'));
-
-    const flatEnglish = flatten(english);
-
-    const apiKey = 'AIzaSyBiqNTundSKFjAJnSb4wSVLDU6w0Kv651M';
-
-    // const tmfc = new TranslateMessageFormatCompiler();
-
-    for (const key of Object.keys(flatEnglish)) {
-        if (!flatGerman[key]) {
-            const value = flatEnglish[key];
-            const apiUrl = `https://www.googleapis.com/language/translate/v2?key=${apiKey}&q=${value}&source=en&target=de`;
-
-            const response = await axios.get(apiUrl);
-            let translated = flatEnglish[key];
-            try {
-                translated = response.data.data.translations[0].translatedText;
-            } catch (e) {}
-            console.log('missing: ' + key + ' -> ' + flatEnglish[key] + ' -> ' + translated);
-            flatGerman[key] = translated;
-        }
-    }
-
-    for (let key of Object.keys(flatGerman)) {
-        if (!flatEnglish[key]) {
-            delete flatGerman[key];
-        }
-    }
-
-    for (let key of Object.keys(flatGerman)) {
-        try {
-            //   const result = tmfc.compile(flatGerman[key], 'de');
-            //   console.log(result);
-        } catch (e) {
-            console.error('ERROR AT: ', flatGerman[key]);
-        }
-    }
-
-    for (let key of Object.keys(flatEnglish)) {
-        try {
-            //   const result = tmfc.compile(flatEnglish[key], 'de');
-            //   console.log(result);
-        } catch (e) {
-            console.error('ERROR AT: ', flatEnglish[key]);
-        }
-    }
-
-    const mergedGerman = { ...flatEnglish, ...flatGerman };
-
-    const finalGerman = unflatten(mergedGerman);
-
-    fs.writeFileSync('./../../apps/red-ui/src/assets/i18n/de.json', JSON.stringify(finalGerman));
-
-    const js: any = {
-        resources: {
-            redaction: {},
-        },
-        sourceLanguage: 'en-US',
-        targetLanguage: 'de-DE',
-    };
-
-    for (const key of Object.keys(flatEnglish)) {
-        js.resources.redaction[`${key}`] = {
-            source: flatEnglish[`${key}`],
-            target: flatGerman[`${key}`],
-        };
-    }
-
-    xliff.jsToXliff12(js, (err: any, res: any) => {
-        fs.writeFileSync('./redaction-en-to-de.xliff', res);
-    });
-
-    const xliffImport = fs.readFileSync('./import.xliff', 'utf-8');
-    xliff.xliff12ToJs(xliffImport, (err: any, res: any) => {
-        const ns = res.resources.redaction;
-
-        const importGerman: any = {};
-        const importEnglish: any = {};
-
-        for (let key of Object.keys(ns)) {
-            importGerman[key] = ns[key].target;
-            importEnglish[key] = ns[key].source;
-        }
-
-        const importReadyEnglish = unflatten(importEnglish);
-        const importReadyGerman = unflatten(importGerman);
-
-        console.log(importReadyEnglish);
-        console.log(importReadyGerman);
-    });
-}
-
-execute().then();

tools/smartcat/files/redaction-en-to-de.xliff

@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="utf-8"?>
+<xliff xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+       xsi:schemaLocation="urn:oasis:names:tc:xliff:document:1.2 http://docs.oasis-open.org/xliff/v1.2/os/xliff-core-1.2-strict.xsd"
+       xmlns="urn:oasis:names:tc:xliff:document:1.2" version="1.2">
+    <file original="redaction" datatype="plaintext" source-language="en-US" target-language="de">
+        <body>
+            <trans-unit id="account-settings" approved="no">
+                <source>Account Settingss</source>
+                <target state="needs-review-translation">󠀰󠀨󠀡󠀬󠀱Account Einstellungen1󠀲</target>
+            </trans-unit>
+            <trans-unit id="something.else" approved="no">
+                <source>Something1</source>
+                <target state="needs-review-translation">󠀰󠀨󠀡󠀭󠀱Something2󠀲</target>
+            </trans-unit>
+            <trans-unit id="something.new" approved="no">
+                <source>Something3</source>
+                <target state="needs-review-translation">󠀰󠀨󠀡󠀮󠀱11233󠀲</target>
+            </trans-unit>
+            <trans-unit id="abc" approved="yes">
+                <source>aa</source>
+                <target state="translated">󠀰󠀨󠀡󠀯󠀱i win at this󠀲</target>
+            </trans-unit>
+        </body>
+    </file>
+</xliff>

tools/smartcat/package.json

@@ -0,0 +1,6 @@
+{
+  "name": "smartcat",
+  "main": "src/index.ts",
+  "version": "1.0.0",
+  "dependencies": {}
+}

tools/smartcat/src/constants.ts

@@ -0,0 +1,18 @@
+import { Language } from './types';
+
+export const languages: Language[] = [
+    { code: 'en', name: 'English', key: 'source' },
+    { code: 'de', name: 'German', key: 'target' },
+];
+
+export const UPDATED_XLIFF_PATH = './tools/smartcat/files/updated.xliff';
+
+/** Test account credentials */
+export const ACCOUNT_ID = '13d238b1-87df-4918-b47d-06dbff048a9d';
+export const API_KEY = '1_Knm1wlSb0377yUMF6k3apKJEa';
+export const DOCUMENT_ID = 'f32a7d3ef55663b72e7fbe9b_7';
+
+/** IQser credentials */
+// export const ACCOUNT_ID="73a11e51-ef77-4c5b-bd55-9b9b5c610988"
+// export const API_KEY="5_QTAp6WD58kq0gwUMuAtCAidI5"
+// export const DOCUMENT_ID="e4b8f6eb18d3c33a486d6e6d_7"

tools/smartcat/src/index.ts

@@ -0,0 +1,22 @@
+import { smartcatApi } from './smartcat-api';
+import { generateXliff, updateLocalFiles, xliffToJson } from './utils';
+
+async function execute(): Promise<void> {
+    const downloadedXliff = await smartcatApi.download();
+    const parsedXliff = await xliffToJson(downloadedXliff);
+    const translations = await updateLocalFiles(parsedXliff);
+    await generateXliff(translations);
+    await smartcatApi.upload();
+
+    /*
+     * TODO:
+     *  Cleanup downloaded / generated files
+     *  Handle placeholders
+     *  See what's wrong with German translations
+     *  ** Handle approved translations
+     * */
+}
+
+execute().then(() => {
+    console.log('Done!');
+});

tools/smartcat/src/smartcat-api.ts

@@ -0,0 +1,49 @@
+/* Personal */
+import axios from 'axios';
+import FormData from 'form-data';
+import fs from 'fs';
+import { ACCOUNT_ID, API_KEY, DOCUMENT_ID, UPDATED_XLIFF_PATH } from './constants';
+
+const BASE_URL = 'https://smartcat.ai/api/integration/v1';
+
+const Headers = {
+    Authorization: `Basic ${Buffer.from(`${ACCOUNT_ID}:${API_KEY}`).toString('base64')}`,
+};
+
+export const smartcatApi = {
+    async createExport(): Promise<string> {
+        console.log('Requesting export...');
+        const url = `${BASE_URL}/document/export?documentIds=${DOCUMENT_ID}&type=DocumentWithMetadata`;
+        const response = await axios.post(url, null, { headers: Headers });
+        return response.data.id;
+    },
+
+    async download(): Promise<string> {
+        const taskId = await this.createExport();
+        console.log(`Got task id: ${taskId}`);
+
+        console.log('Waiting for export to finish...');
+        await new Promise(f => setTimeout(f, 2000));
+
+        console.log('Requesting download...');
+        const url = `${BASE_URL}/document/export/${taskId}`;
+        const response = await axios.get(url, { headers: Headers });
+        console.log('Downloaded translation');
+        return response.data;
+    },
+
+    async upload() {
+        try {
+            const url = `${BASE_URL}/document/update?documentId=${DOCUMENT_ID}`;
+            const formData = new FormData();
+            formData.append('file', fs.createReadStream(UPDATED_XLIFF_PATH));
+            const response = await axios.putForm(url, formData, {
+                headers: Headers,
+            });
+            console.log(response.data);
+        } catch (e) {
+            console.error(e);
+            console.error(`${e.response?.status} - ${e.response?.statusText}`);
+        }
+    },
+};

tools/smartcat/src/types.ts

@@ -0,0 +1,5 @@
+export type ParsedXliff = Record<string, { source: string; target: string; additionalAttributes: Record<string, string> }>;
+
+export type Language = { code: string; name: string; key: string };
+
+export type Translation = { content: string; lang: Language };

tools/smartcat/src/utils.ts

@@ -0,0 +1,101 @@
+import * as xliffConverter from 'xliff';
+import { ParsedXliff, Translation } from './types';
+import { languages, UPDATED_XLIFF_PATH } from './constants';
+import fs from 'fs';
+
+export async function updateLocalFiles(downloadedXliff: ParsedXliff): Promise<Translation[]> {
+    const translations: Translation[] = languages.map(lang => ({
+        content: flattenJson(JSON.parse(fs.readFileSync(`./apps/red-ui/src/assets/i18n/${lang.code}.json`, 'utf8'))),
+        lang,
+    }));
+    const source = translations[0]; // flat English
+    const xliffKeys = Object.keys(downloadedXliff);
+
+    // Update json files with potential new translations
+    for (const key of xliffKeys) {
+        if (source.content[key] !== undefined) {
+            for (const translation of translations) {
+                const newValue = downloadedXliff[key][translation.lang.key];
+                const oldValue = translation.content[key];
+                if (newValue && newValue !== translation.content[key]) {
+                    translation.content[key] = newValue;
+                    console.log(`Updated ${translation.lang.name} translation for [${key}].`);
+                    console.log(`Old value: ${oldValue}, new value: ${newValue}`);
+                }
+            }
+        } else {
+            console.log(`Key [${key}] doesn't exist in en.json - probably was removed.`);
+        }
+    }
+
+    const unflattenedTranslations = translations.map(translation => ({
+        content: unflattenJson(translation.content),
+        lang: translation.lang,
+    }));
+
+    console.log('Updating translation files...');
+    unflattenedTranslations.forEach(translation => {
+        fs.writeFileSync(
+            `./apps/red-ui/src/assets/i18n/${translation.lang.code}.json`,
+            JSON.stringify(translation.content, null, 2) + '\n',
+        );
+    });
+    console.log('Updated translation files!');
+
+    return translations;
+}
+
+export async function xliffToJson(xliff: string): Promise<ParsedXliff> {
+    const data = await xliffConverter.xliff12ToJs(xliff);
+    return data.resources.redaction;
+}
+
+export async function generateXliff(translations: Translation[]): Promise<void> {
+    console.log('Generating new XLIFF...');
+
+    const xliff = await xliffConverter.createxliff12('en-US', 'de-DE', translations[0].content, translations[1].content, 'redaction');
+    fs.writeFileSync(UPDATED_XLIFF_PATH, xliff);
+
+    console.log(`Writing new XLIFF to ${UPDATED_XLIFF_PATH} succeeded!`);
+}
+
+export function flattenJson(data: any) {
+    const result: any = {};
+
+    function recurse(cur: any, prop: any) {
+        if (Object(cur) !== cur) {
+            result[prop] = cur;
+        } else if (Array.isArray(cur)) {
+            let l = 0;
+            for (let i = 0, l = cur.length; i < l; i++) recurse(cur[i], prop + '[' + i + ']');
+            if (l === 0) result[prop] = [];
+        } else {
+            let isEmpty = true;
+            for (const p in cur) {
+                isEmpty = false;
+                recurse(cur[p], prop ? prop + '.' + p : p);
+            }
+            if (isEmpty && prop) result[prop] = {};
+        }
+    }
+
+    recurse(data, '');
+    return result;
+}
+
+export function unflattenJson(data: any) {
+    if (Object(data) !== data || Array.isArray(data)) return data;
+    const regex = /\.?([^.\[\]]+)|\[(\d+)\]/g,
+        resultholder: any = {};
+    for (const p in data) {
+        let cur: any = resultholder,
+            prop = '',
+            m: any;
+        while ((m = regex.exec(p))) {
+            cur = cur[prop] || (cur[prop] = m[2] ? [] : {});
+            prop = m[2] || m[1];
+        }
+        cur[prop] = data[p];
+    }
+    return resultholder[''] || resultholder;
+}

tools/smartcat/tsconfig.json

@@ -0,0 +1,11 @@
+{
+  "compilerOptions": {
+    "module": "commonjs",
+    "esModuleInterop": true,
+    "target": "es6",
+    "moduleResolution": "node",
+    "sourceMap": true,
+    "outDir": "dist"
+  },
+  "lib": ["es2015"]
+}