cloudron-box/scripts/weblate

#!/usr/bin/env node

import { execFileSync, execSync } from 'child_process';
import fs from 'fs';
import path from 'path';
import readline from 'readline';
import { program } from 'commander';
import superagent from '@cloudron/superagent';

const WEBLATE_URL = 'https://translate.cloudron.io';
const PROJECT = 'cloudron';
const COMPONENT = 'dashboard';
const TRANSLATION_DIR = path.resolve('./dashboard/public/translation');
const SRC_DIRS = [path.resolve('./dashboard/src'), path.resolve('./src')];

function getToken() {
    const token = process.env.WEBLATE_API_KEY;
    if (!token) {
        console.error('Error: WEBLATE_API_KEY environment variable is not set.');
        console.error('Get your token from https://translate.cloudron.io/accounts/profile/#api');
        process.exit(1);
    }
    return token;
}

function escapeRegex(str) {
    return str.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
}

function getNestedValue(obj, dottedKey) {
    return dottedKey.split('.').reduce((current, key) => current && current[key], obj);
}

function flattenObject(obj, prefix = '') {
    const result = {};
    for (const key of Object.keys(obj)) {
        const fullKey = prefix ? `${prefix}.${key}` : key;
        if (typeof obj[key] === 'object' && obj[key] !== null && !Array.isArray(obj[key])) {
            Object.assign(result, flattenObject(obj[key], fullKey));
        } else {
            result[fullKey] = obj[key];
        }
    }
    return result;
}

async function listFiles(dir) {
    let result = [];
    const entries = await fs.promises.readdir(dir, { withFileTypes: true });
    for (const entry of entries) {
        const fullPath = path.join(dir, entry.name);
        if (entry.isDirectory()) {
            result = result.concat(await listFiles(fullPath));
        } else if (entry.isFile()) {
            result.push(fullPath);
        }
    }
    return result;
}

async function getSourceFiles() {
    let files = [];
    for (const dir of SRC_DIRS) {
        if (!fs.existsSync(dir)) continue;
        files = files.concat(await listFiles(dir));
    }
    return files.filter(f => f.endsWith('.vue') || f.endsWith('.js'));
}

function getFindUsageFilesForKey(key) {
    try {
        const output = execFileSync('git', ['grep', '-l', '-F', '--untracked', key], { encoding: 'utf-8' });
        return output.trim().length ? output.trim().split('\n') : [];
    } catch (error) {
        if (error.status === 1) return [];
        throw error;
    }
}

async function yesno(question) {
    const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
    return new Promise((resolve) => {
        rl.question(question + ' [y/n] ', (answer) => {
            rl.close();
            resolve(answer.trim().toLowerCase() === 'y');
        });
    });
}

function readAllTranslations(key) {
    const translationFiles = fs.readdirSync(TRANSLATION_DIR).filter(f => f.endsWith('.json'));
    const translations = {};
    for (const file of translationFiles) {
        const lang = path.basename(file, '.json');
        const data = JSON.parse(fs.readFileSync(path.join(TRANSLATION_DIR, file), 'utf-8'));
        const value = getNestedValue(data, key);
        if (value !== undefined) translations[lang] = value;
    }
    return translations;
}

async function findTranslations(query) {
    const normalizedQuery = query.trim().toLowerCase();
    if (!normalizedQuery) {
        console.error('Please provide a non-empty search text.');
        process.exit(1);
    }

    const translationFiles = fs.readdirSync(TRANSLATION_DIR).filter(f => f.endsWith('.json'));
    const matches = [];

    for (const file of translationFiles) {
        const lang = path.basename(file, '.json');
        const data = JSON.parse(fs.readFileSync(path.join(TRANSLATION_DIR, file), 'utf-8'));
        const flat = flattenObject(data);

        for (const [key, value] of Object.entries(flat)) {
            if (typeof value !== 'string') continue;
            if (!value.toLowerCase().includes(normalizedQuery)) continue;
            matches.push({ key, lang, value });
        }
    }

    if (matches.length === 0) {
        console.error(`No matches found for "${query}".`);
        process.exit(1);
    }

    const groupedByKey = new Map();
    for (const match of matches) {
        if (!groupedByKey.has(match.key)) groupedByKey.set(match.key, []);
        groupedByKey.get(match.key).push(match);
    }

    const keyUsage = new Map();
    for (const key of groupedByKey.keys()) {
        keyUsage.set(key, getFindUsageFilesForKey(key));
    }

    const sortedKeys = [...groupedByKey.keys()].sort();
    console.log(`Found ${matches.length} match(es) across ${sortedKeys.length} key(s):\n`);
    for (const key of sortedKeys) {
        console.log(`  ${key}`);
        const keyMatches = groupedByKey.get(key).sort((a, b) => a.lang.localeCompare(b.lang));
        for (const { lang, value } of keyMatches) {
            const snippet = value.length > 120 ? value.slice(0, 120) + '...' : value;
            console.log(`    [${lang}] ${snippet}`);
        }
        const usedIn = keyUsage.get(key);
        if (usedIn.length > 0) {
            console.log('    used in:');
            for (const file of usedIn) console.log(`      - ${file}`);
        } else {
            console.log('    used in: (no matches in repository)');
        }
        console.log();
    }
}

async function deleteUnit(token, key) {
    const result = await superagent
        .get(`${WEBLATE_URL}/api/units/`)
        .set('Authorization', `Token ${token}`)
        .query({ q: `language:en AND component:${COMPONENT} AND key:${key}` });

    if (!result.body.results || result.body.results.length === 0) {
        console.error(`  Warning: could not find unit for key "${key}" in Weblate.`);
        return false;
    }

    const id = result.body.results[0].id;
    console.log(`  Deleting unit ${id}...`);
    await superagent
        .del(`${WEBLATE_URL}/api/units/${id}/`)
        .set('Authorization', `Token ${token}`);
    return true;
}

// ---- download ----

function downloadTranslations() {
    console.log('Downloading translations...');
    const zipPath = path.resolve('./lang.zip');
    execSync(`curl -s "${WEBLATE_URL}/api/components/${PROJECT}/${COMPONENT}/file/" -o "${zipPath}"`);
    console.log('Unpacking...');
    execSync(`unzip -jo "${zipPath}" -d "${TRANSLATION_DIR}"`);
    fs.unlinkSync(zipPath);
    console.log('Done. Translations downloaded.');
}

// ---- get ----

function getTranslation(key) {
    downloadTranslations();

    const translations = readAllTranslations(key);

    if (Object.keys(translations).length === 0) {
        console.error(`Key "${key}" not found in any language file.`);
        process.exit(1);
    }

    for (const [lang, value] of Object.entries(translations)) {
        console.log(`  ${lang}: ${value}`);
    }
}

// ---- add ----

async function addTranslation(key, value) {
    const token = getToken();

    console.log(`\nAdding key "${key}" with value "${value}" in Weblate...`);
    await superagent
        .post(`${WEBLATE_URL}/api/translations/${PROJECT}/${COMPONENT}/en/units/`)
        .set('Authorization', `Token ${token}`)
        .send({ key, value: [value] });

    console.log('Downloading updated translation files...');
    downloadTranslations();

    console.log('Done.');
}

// ---- update ----

async function updateTranslation(key, value) {
    const token = getToken();

    console.log(`\nLooking up key "${key}" in Weblate...`);

    const result = await superagent
        .get(`${WEBLATE_URL}/api/units/`)
        .set('Authorization', `Token ${token}`)
        .query({ q: `language:en AND component:${COMPONENT} AND key:${key}` });

    if (!result.body.results || result.body.results.length === 0) {
        console.error(`Error: key "${key}" not found in Weblate.`);
        process.exit(1);
    }

    const unit = result.body.results[0];
    console.log(`  Found unit ${unit.id}, current value: "${unit.source[0]}"`);
    console.log(`  Updating to: "${value}"...`);

    await superagent
        .patch(`${WEBLATE_URL}/api/units/${unit.id}/`)
        .set('Authorization', `Token ${token}`)
        .send({ target: [value], state: 20 /* numeric code for translated */ });

    console.log('Downloading updated translation files...');
    downloadTranslations();

    console.log('Done.');
}

// ---- del ----

async function delTranslation(key) {
    const token = getToken();

    console.log(`\nDeleting key "${key}" from Weblate...`);
    if (await deleteUnit(token, key)) {
        console.log('Downloading updated translation files...');
        downloadTranslations();
        console.log('Done.');
    } else {
        process.exit(1);
    }
}

// ---- duplicates ----

async function findDuplicates() {
    const enPath = path.join(TRANSLATION_DIR, 'en.json');
    const data = JSON.parse(fs.readFileSync(enPath, 'utf-8'));
    const flat = flattenObject(data);
    const keys = Object.keys(flat);

    // group keys by their value
    const byValue = new Map();
    for (const key of keys) {
        const value = flat[key];
        if (!byValue.has(value)) byValue.set(value, []);
        byValue.get(value).push(key);
    }

    const duplicates = [...byValue.entries()]
        .filter(([, group]) => group.length > 1)
        .sort((a, b) => b[1].length - a[1].length);

    if (duplicates.length === 0) {
        console.log('No duplicate values found.');
        return;
    }

    console.log(`Found ${duplicates.length} values with multiple keys:\n`);
    for (const [value, group] of duplicates) {
        const truncated = value.length > 60 ? value.substring(0, 60) + '...' : value;
        console.log(`  "${truncated}"`);
        for (const key of group) console.log(`    - ${key}`);
        console.log();
    }
}

// ---- unused ----

async function findUnused() {
    const enPath = path.join(TRANSLATION_DIR, 'en.json');
    const data = JSON.parse(fs.readFileSync(enPath, 'utf-8'));
    const flat = flattenObject(data);
    const keys = Object.keys(flat);

    console.log(`Scanning ${keys.length} keys against source files...\n`);

    const files = await getSourceFiles();
    const fileContents = await Promise.all(files.map(f => fs.promises.readFile(f, 'utf-8')));

    const unused = keys.filter(key => !fileContents.some(content => content.includes(key)));

    if (unused.length === 0) {
        console.log('No unused keys found.');
        return;
    }

    console.log(`Found ${unused.length} unused key(s):\n`);
    for (const key of unused) console.log(`  ${key}`);
    console.log();
}

// ---- rename ----

async function renameInSourceCode(oldKey, newKey) {
    const files = await getSourceFiles();
    let totalReplacements = 0;
    const modifiedFiles = [];

    for (const file of files) {
        let content = await fs.promises.readFile(file, 'utf-8');
        if (!content.includes(oldKey)) continue;

        const relPath = path.relative(process.cwd(), file);
        let fileModified = false;

        // safe: inside $t('...') or $t("...")
        const safePattern = new RegExp(`(\\$t\\(['"])${escapeRegex(oldKey)}(['",)])`, 'g');
        const safeMatches = content.match(safePattern);
        if (safeMatches) {
            content = content.replace(safePattern, `$1${newKey}$2`);
            totalReplacements += safeMatches.length;
            fileModified = true;
            console.log(`  ${relPath}: ${safeMatches.length} $t() replacement(s)`);
        }

        // suspicious: key still appears outside $t() after safe replacements
        if (content.includes(oldKey)) {
            const lines = content.split('\n');
            for (let i = 0; i < lines.length; i++) {
                if (!lines[i].includes(oldKey)) continue;
                console.log(`\n  Suspicious match in ${relPath}:${i + 1}:`);
                const start = Math.max(0, i - 1);
                const end = Math.min(lines.length, i + 2);
                for (let j = start; j < end; j++) {
                    const marker = j === i ? '>>>' : '   ';
                    console.log(`    ${marker} ${j + 1}: ${lines[j]}`);
                }
                const replace = await yesno('  Replace this occurrence?');
                if (replace) {
                    lines[i] = lines[i].replaceAll(oldKey, newKey);
                    totalReplacements++;
                    fileModified = true;
                }
            }
            content = lines.join('\n');
        }

        if (fileModified) {
            await fs.promises.writeFile(file, content, 'utf-8');
            modifiedFiles.push(relPath);
        }
    }

    return { totalReplacements, modifiedFiles };
}

async function renameTranslation(oldKey, newKey) {
    const token = getToken();

    // Step 1: download fresh translations
    console.log(`\nStep 1: Downloading latest translation files...`);
    downloadTranslations();

    // Step 2: rename in source code
    console.log(`\nStep 2: Renaming in source code...`);

    const { totalReplacements, modifiedFiles } = await renameInSourceCode(oldKey, newKey);
    console.log(`\n  ${totalReplacements} replacement(s) in ${modifiedFiles.length} file(s).`);

    if (totalReplacements === 0) {
        console.error(`\nNo occurrences of "${oldKey}" found in source code. Aborting.`);
        process.exit(1);
    }

    // Step 3: read existing translations from all JSON files
    console.log(`\nStep 3: Reading existing translations for key "${oldKey}"...`);

    const translations = readAllTranslations(oldKey);
    for (const [lang, value] of Object.entries(translations)) {
        console.log(`  ${lang}: "${typeof value === 'string' ? value.substring(0, 60) : value}"`);
    }

    if (!translations.en) {
        console.error(`Error: key "${oldKey}" not found in en.json. Aborting.`);
        process.exit(1);
    }

    console.log(`\nFound translations in ${Object.keys(translations).length} languages.`);

    // Step 4: add the new key in Weblate for each language
    console.log(`\nStep 4: Adding new key "${newKey}" in Weblate...`);

    console.log(`  Adding en (source)...`);
    await superagent
        .post(`${WEBLATE_URL}/api/translations/${PROJECT}/${COMPONENT}/en/units/`)
        .set('Authorization', `Token ${token}`)
        .send({ key: newKey, value: [translations.en] });

    for (const [lang, value] of Object.entries(translations)) {
        if (lang === 'en') continue;
        console.log(`  Adding ${lang}...`);
        try {
            await superagent
                .post(`${WEBLATE_URL}/api/translations/${PROJECT}/${COMPONENT}/${lang}/units/`)
                .set('Authorization', `Token ${token}`)
                .send({ key: newKey, value: [value], state: 20 });
        } catch (err) {
            console.warn(`  Warning: failed to add ${lang}: ${err.message}`);
        }
    }

    // Step 5: delete the old source unit from Weblate
    console.log(`\nStep 5: Deleting old key "${oldKey}" from Weblate...`);
    await deleteUnit(token, oldKey);

    // Step 6: re-download translations
    console.log('\nStep 6: Downloading updated translation files...');
    downloadTranslations();

    console.log(`\nDone! Renamed "${oldKey}" -> "${newKey}".`);
}

// ---- CLI ----

program
    .name('weblate')
    .description('Weblate translation management tools');

program
    .command('download')
    .description('Download latest translations from Weblate')
    .action(() => downloadTranslations());

program
    .command('get <key>')
    .description('Show the value of a key across all languages')
    .action((key) => getTranslation(key));

program
    .command('find <text>')
    .description('Reverse-lookup translation text across languages and show matching key paths')
    .action(async (text) => await findTranslations(text));

program
    .command('add <key> <value>')
    .description('Add a new English source string in Weblate')
    .action(async (key, value) => await addTranslation(key, value));

program
    .command('update <key> <value>')
    .description('Update the English value of an existing key in Weblate')
    .action(async (key, value) => await updateTranslation(key, value));

program
    .command('del <key>')
    .description('Delete a translation key from Weblate (all languages)')
    .action(async (key) => await delTranslation(key));

program
    .command('rename <oldKey> <newKey>')
    .description('Rename a translation key across Weblate and source code')
    .action(async (oldKey, newKey) => await renameTranslation(oldKey, newKey));

program
    .command('duplicates')
    .description('Find English keys that share the same value')
    .action(async () => await findDuplicates());

program
    .command('unused')
    .description('Find translation keys not referenced in source code')
    .action(async () => await findUnused());

program.parse();