cloudron-box/src/backuptask.js

'use strict';

exports = module.exports = {
    fullBackup,
    appBackup,

    restore,

    downloadApp,
    backupApp,

    downloadMail,

    upload,
};

const apps = require('./apps.js'),
    assert = require('node:assert'),
    backupFormat = require('./backupformat.js'),
    backups = require('./backups.js'),
    backupTargets = require('./backuptargets.js'),
    BoxError = require('./boxerror.js'),
    constants = require('./constants.js'),
    DataLayout = require('./datalayout.js'),
    database = require('./database.js'),
    debug = require('debug')('box:backuptask'),
    df = require('./df.js'),
    locks = require('./locks.js'),
    path = require('node:path'),
    paths = require('./paths.js'),
    { Readable } = require('node:stream'),
    safe = require('safetydance'),
    services = require('./services.js'),
    shell = require('./shell.js')('backuptask'),
    stream = require('stream/promises');

const BACKUP_UPLOAD_CMD = path.join(__dirname, 'scripts/backupupload.js');

function addFileExtension(backupTarget, remotePath) {
    assert.strictEqual(typeof backupTarget, 'object');
    assert.strictEqual(typeof remotePath, 'string');

    const ext = backupFormat.api(backupTarget.format).getFileExtension(!!backupTarget.encyption);
    return remotePath + ext;
}

async function checkPreconditions(backupTarget, dataLayout) {
    assert.strictEqual(typeof backupTarget, 'object');
    assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');

    // check mount status before uploading
    const status = await backupTargets.ensureMounted(backupTarget);
    debug(`checkPreconditions: mount point status is ${JSON.stringify(status)}`);
    if (status.state !== 'active') throw new BoxError(BoxError.MOUNT_ERROR, `Backup endpoint is not active: ${status.message}`);

    // check availabe size. this requires root for df to work
    const available = await backupTargets.storageApi(backupTarget).getAvailableSize(backupTarget.config);
    let used = 0;
    for (const localPath of dataLayout.localPaths()) {
        debug(`checkPreconditions: getting disk usage of ${localPath}`);
        // du can error when files go missing as it is computing the size. it still prints some size anyway
        // to match df output in getAvailableSize() we must use disk usage size here and not apparent size
        const [duError, result] = await safe(shell.spawn('du', [ '--dereference-args', '--summarize', '--block-size=1', '--exclude=*.lock', '--exclude=dovecot.list.index.log.*', localPath], { encoding: 'utf8' }));
        if (duError) debug(`checkPreconditions: du error for ${localPath}. code: ${duError.code} stderror: ${duError.stderr}`);
        used += parseInt(duError ? duError.stdout : result, 10);
    }

    debug(`checkPreconditions: total required=${used} available=${available}`);

    const needed = 0.6 * used + (1024 * 1024 * 1024); // check if there is atleast 1GB left afterwards. aim for 60% because rsync/tgz won't need full 100%
    if (available <= needed) throw new BoxError(BoxError.FS_ERROR, `Not enough disk space for backup. Needed: ${df.prettyBytes(needed)} Available: ${df.prettyBytes(available)}`);
}

async function uploadBackupInfo(backupTarget, remotePath, integrityMap) {
    const sortedIntegrityMap = [...integrityMap.entries()].sort(([a], [b]) => a < b); // for readability, order the entries
    const integrityDataJsonString = JSON.stringify(Object.fromEntries(sortedIntegrityMap), null, 2);
    const integrityDataStream = Readable.from(integrityDataJsonString);
    const integrityUploader = await backupTargets.storageApi(backupTarget).upload(backupTarget.config, `${remotePath}.backupinfo`);
    await stream.pipeline(integrityDataStream, integrityUploader.stream);
    await integrityUploader.finish();

    return await crypto.sign(null /* algorithm */, integrityDataJsonString, backupTarget.integrityKeyPair.privateKey);
}

// this function is called via backupupload (since it needs root to traverse app's directory)
async function upload(remotePath, targetId, dataLayoutString, progressCallback) {
    assert.strictEqual(typeof remotePath, 'string');
    assert.strictEqual(typeof targetId, 'string');
    assert.strictEqual(typeof dataLayoutString, 'string');
    assert.strictEqual(typeof progressCallback, 'function');

    debug(`upload: path ${remotePath} target ${targetId} dataLayout ${dataLayoutString}`);

    const backupTarget = await backupTargets.get(targetId);
    if (!backupTarget) throw new BoxError(BoxError.NOT_FOUND, 'Backup target not found');

    const dataLayout = DataLayout.fromString(dataLayoutString);

    await checkPreconditions(backupTarget, dataLayout);

    const { stats, integrityMap } = await backupFormat.api(backupTarget.format).upload(backupTarget, remotePath, dataLayout, progressCallback);

    const signature = await uploadBackupInfo(backupTarget, remotePath, integrityMap);
    return { stats, integrity: { signature } };
}

async function download(backupTarget, remotePath, dataLayout, progressCallback) {
    assert.strictEqual(typeof backupTarget, 'object');
    assert.strictEqual(typeof remotePath, 'string');
    assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
    assert.strictEqual(typeof progressCallback, 'function');

    debug(`download: Downloading ${remotePath} of format ${backupTarget.format} (encrypted: ${!!backupTarget.encryption}) to ${dataLayout.toString()}`);

    await backupFormat.api(backupTarget.format).download(backupTarget, remotePath, dataLayout, progressCallback);
}

async function restore(backupTarget, remotePath, progressCallback) {
    assert.strictEqual(typeof backupTarget, 'object');
    assert.strictEqual(typeof remotePath, 'string');
    assert.strictEqual(typeof progressCallback, 'function');

    const boxDataDir = safe.fs.realpathSync(paths.BOX_DATA_DIR);
    if (!boxDataDir) throw new BoxError(BoxError.FS_ERROR, `Error resolving boxdata: ${safe.error.message}`);
    const dataLayout = new DataLayout(boxDataDir, []);

    await download(backupTarget, remotePath, dataLayout, progressCallback);

    debug('restore: download completed, importing database');

    await database.importFromFile(`${dataLayout.localRoot()}/box.mysqldump`);
    debug('restore: database imported');

    await locks.releaseAll(); // clear the locks table in database
}

async function downloadApp(app, restoreConfig, progressCallback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof restoreConfig, 'object');
    assert.strictEqual(typeof progressCallback, 'function');

    const appDataDir = safe.fs.realpathSync(path.join(paths.APPS_DATA_DIR, app.id));
    if (!appDataDir) throw new BoxError(BoxError.FS_ERROR, safe.error.message);
    const dataLayout = new DataLayout(appDataDir, app.storageVolumeId ? [{ localDir: await apps.getStorageDir(app), remoteDir: 'data' }] : []);

    const startTime = new Date();
    let { backupTarget, remotePath } = restoreConfig; // set when importing
    if (!remotePath) {
        const backup = await backups.get(restoreConfig.backupId);
        if (!backup) throw new BoxError(BoxError.BAD_FIELD, 'No such backup');
        remotePath = backup.remotePath;
        backupTarget = await backupTargets.get(backup.targetId);
    }

    await download(backupTarget, remotePath, dataLayout, progressCallback);
    debug('downloadApp: time: %s', (new Date() - startTime)/1000);
}

async function runBackupUpload(uploadConfig, progressCallback) {
    assert.strictEqual(typeof uploadConfig, 'object');
    assert.strictEqual(typeof progressCallback, 'function');

    const { remotePath, backupTarget, dataLayout, progressTag } = uploadConfig;
    assert.strictEqual(typeof remotePath, 'string');
    assert.strictEqual(typeof backupTarget, 'object');
    assert.strictEqual(typeof progressTag, 'string');
    assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');

    // https://stackoverflow.com/questions/48387040/node-js-recommended-max-old-space-size
    const envCopy = Object.assign({}, process.env);
    if (backupTarget.limits?.memoryLimit >= 2*1024*1024*1024) {
        const heapSize = Math.min((backupTarget.limits.memoryLimit/1024/1024) - 256, 8192);
        debug(`runBackupUpload: adjusting heap size to ${heapSize}M`);
        envCopy.NODE_OPTIONS = `--max-old-space-size=${heapSize}`;
    }

    let lastMessage = null; // the script communicates error result as a string
    function onMessage(progress) { // this is { message } or { result }
        if ('message' in progress) return progressCallback({ message: `${progress.message} (${progressTag})` });
        debug(`runBackupUpload: result - ${JSON.stringify(progress)}`);
        lastMessage = progress;
    }

    // do not use debug for logging child output because it already has timestamps via it's own debug
    const [error] = await safe(shell.sudo([ BACKUP_UPLOAD_CMD, remotePath, backupTarget.id, dataLayout.toString() ], { env: envCopy, preserveEnv: true, onMessage, logger: process.stdout.write }));
    if (error && (error.code === null /* signal */ || (error.code !== 0 && error.code !== 50))) { // backuptask crashed
        debug(`runBackupUpload: backuptask crashed`, error);
        throw new BoxError(BoxError.INTERNAL_ERROR, 'Backuptask crashed');
    } else if (error && error.code === 50) { // exited with error
        throw new BoxError(BoxError.EXTERNAL_ERROR, lastMessage.errorMessage);
    }

    return lastMessage.result;
}

async function snapshotBox(progressCallback) {
    assert.strictEqual(typeof progressCallback, 'function');

    progressCallback({ message: 'Snapshotting box' });

    const startTime = new Date();

    await database.exportToFile(`${paths.BOX_DATA_DIR}/box.mysqldump`);
    debug(`snapshotBox: took ${(new Date() - startTime)/1000} seconds`);
}

async function uploadBoxSnapshot(backupTarget, progressCallback) {
    assert.strictEqual(typeof backupTarget, 'object');
    assert.strictEqual(typeof progressCallback, 'function');

    await snapshotBox(progressCallback);

    const remotePath = addFileExtension(backupTarget, `snapshot/box`);

    const boxDataDir = safe.fs.realpathSync(paths.BOX_DATA_DIR);
    if (!boxDataDir) throw new BoxError(BoxError.FS_ERROR, `Error resolving boxdata: ${safe.error.message}`);

    const uploadConfig = {
        remotePath,
        backupTarget,
        dataLayout: new DataLayout(boxDataDir, []),
        progressTag: 'box'
    };

    progressCallback({ message: 'Uploading box snapshot' });

    const startTime = new Date();

    const { stats, integrity } = await runBackupUpload(uploadConfig, progressCallback);

    debug(`uploadBoxSnapshot: took ${(new Date() - startTime)/1000} seconds`);

    await backupTargets.setSnapshotInfo(backupTarget, 'box', { timestamp: new Date().toISOString() });

    return { stats, integrity };
}

async function copy(backupTarget, srcRemotePath, destRemotePath, progressCallback) {
    assert.strictEqual(typeof backupTarget, 'object');
    assert.strictEqual(typeof srcRemotePath, 'string');
    assert.strictEqual(typeof destRemotePath, 'string');
    assert.strictEqual(typeof progressCallback, 'function');

    const startTime = new Date();
    const [copyError] = await safe(backupTargets.storageApi(backupTarget).copy(backupTarget.config, srcRemotePath, destRemotePath, progressCallback));
    if (copyError) {
        debug(`copy: copied to ${destRemotePath} errored. error: ${copyError.message}`);
        throw copyError;
    }
    debug(`copy: copied successfully to ${destRemotePath}. Took ${(new Date() - startTime)/1000} seconds`);

    const [copyChecksumError] = await safe(backupTargets.storageApi(backupTarget).copy(backupTarget.config, `${srcRemotePath}.backupinfo`, `${destRemotePath}.backupinfo`, progressCallback));
    if (copyChecksumError) {
        debug(`copy: copied to ${destRemotePath} errored. error: ${copyChecksumError.message}`);
        throw copyChecksumError;
    }
    debug(`copy: copied backupinfo successfully to ${destRemotePath}.backupinfo`);
}

async function backupBox(backupTarget, dependsOn, tag, options, progressCallback) {
    assert.strictEqual(typeof backupTarget, 'object');
    assert(Array.isArray(dependsOn));
    assert.strictEqual(typeof tag, 'string');
    assert.strictEqual(typeof options, 'object');
    assert.strictEqual(typeof progressCallback, 'function');

    const { stats, integrity } = await uploadBoxSnapshot(backupTarget, progressCallback);

    const remotePath = addFileExtension(backupTarget, `${tag}/box_v${constants.VERSION}`);

    debug(`backupBox: rotating to id ${remotePath}`);

    const data = {
        remotePath,
        encryptionVersion: backupTarget.encryption ? 2 : null,
        packageVersion: constants.VERSION,
        type: backups.BACKUP_TYPE_BOX,
        state: backups.BACKUP_STATE_CREATING,
        identifier: backups.BACKUP_IDENTIFIER_BOX,
        dependsOn,
        manifest: null,
        preserveSecs: options.preserveSecs || 0,
        appConfig: null,
        targetId: backupTarget.id,
        stats,
        integrity
    };

    const id = await backups.add(data);
    const snapshotPath = addFileExtension(backupTarget, 'snapshot/box');
    const [error] = await safe(copy(backupTarget, snapshotPath, remotePath, progressCallback));
    const state = error ? backups.BACKUP_STATE_ERROR : backups.BACKUP_STATE_NORMAL;
    await backups.setState(id, state);
    if (error) throw error;

    return id;
}

async function snapshotApp(app, progressCallback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof progressCallback, 'function');

    const startTime = new Date();
    progressCallback({ message: `Snapshotting app ${app.fqdn}` });

    await apps.writeConfig(app);
    await services.backupAddons(app, app.manifest.addons);

    debug(`snapshotApp: ${app.fqdn} took ${(new Date() - startTime)/1000} seconds`);
}

async function uploadAppSnapshot(backupTarget, app, progressCallback) {
    assert.strictEqual(typeof backupTarget, 'object');
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof progressCallback, 'function');

    await snapshotApp(app, progressCallback);

    const remotePath = addFileExtension(backupTarget, `snapshot/app_${app.id}`);
    const appDataDir = safe.fs.realpathSync(path.join(paths.APPS_DATA_DIR, app.id));
    if (!appDataDir) throw new BoxError(BoxError.FS_ERROR, `Error resolving appsdata: ${safe.error.message}`);

    const dataLayout = new DataLayout(appDataDir, app.storageVolumeId ? [{ localDir: await apps.getStorageDir(app), remoteDir: 'data' }] : []);

    progressCallback({ message: `Uploading app snapshot ${app.fqdn}`});

    const uploadConfig = {
        remotePath,
        backupTarget,
        dataLayout,
        progressTag: app.fqdn
    };

    const startTime = new Date();

    const { stats, integrity } = await runBackupUpload(uploadConfig, progressCallback);

    debug(`uploadAppSnapshot: ${app.fqdn} uploaded to ${remotePath}. ${(new Date() - startTime)/1000} seconds`);

    await backupTargets.setSnapshotInfo(backupTarget, app.id, { timestamp: new Date().toISOString(), manifest: app.manifest });

    return { stats, integrity };
}

async function backupAppWithTag(app, backupTarget, tag, options, progressCallback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof backupTarget, 'object');
    assert.strictEqual(typeof tag, 'string');
    assert.strictEqual(typeof options, 'object');
    assert.strictEqual(typeof progressCallback, 'function');

    if (!apps.canBackupApp(app)) { // if we cannot backup, reuse it's most recent backup
        const lastKnownGoodAppBackup = await backups.getLatestInTargetByIdentifier(app.id, backupTarget.id);
        if (lastKnownGoodAppBackup === null) return null; // no backup to re-use
        return lastKnownGoodAppBackup.id;
    }

    const { stats, integrity } = await uploadAppSnapshot(backupTarget, app, progressCallback);

    const manifest = app.manifest;
    const remotePath = addFileExtension(backupTarget, `${tag}/app_${app.fqdn}_v${manifest.version}`);

    debug(`backupAppWithTag: rotating ${app.fqdn} to path ${remotePath}`);

    const data = {
        remotePath,
        encryptionVersion: backupTarget.encryption ? 2 : null,
        packageVersion: manifest.version,
        type: backups.BACKUP_TYPE_APP,
        state: backups.BACKUP_STATE_CREATING,
        identifier: app.id,
        dependsOn: [],
        manifest,
        preserveSecs: options.preserveSecs || 0,
        appConfig: app,
        targetId: backupTarget.id,
        stats,
        integrity
    };

    const id = await backups.add(data);
    const snapshotPath = addFileExtension(backupTarget, `snapshot/app_${app.id}`);
    const [error] = await safe(copy(backupTarget,  snapshotPath, remotePath, progressCallback));
    const state = error ? backups.BACKUP_STATE_ERROR : backups.BACKUP_STATE_NORMAL;
    await backups.setState(id, state);
    if (error) throw error;

    return id;
}

async function backupApp(app, backupTarget, options, progressCallback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof backupTarget, 'object');
    assert.strictEqual(typeof options, 'object');
    assert.strictEqual(typeof progressCallback, 'function');

    let backupId = null;
    await locks.wait(`${locks.TYPE_APP_BACKUP_PREFIX}${app.id}`);
    if (options.snapshotOnly) {
        await snapshotApp(app, progressCallback);
    } else {
        const tag = (new Date()).toISOString().replace(/[T.]/g, '-').replace(/[:Z]/g,'');
        backupId = await backupAppWithTag(app, backupTarget, tag, options, progressCallback);
    }
    await locks.release(`${locks.TYPE_APP_BACKUP_PREFIX}${app.id}`);

    return backupId;
}

async function uploadMailSnapshot(backupTarget, progressCallback) {
    assert.strictEqual(typeof backupTarget, 'object');
    assert.strictEqual(typeof progressCallback, 'function');

    const remotePath = addFileExtension(backupTarget, 'snapshot/mail');

    const mailDataDir = safe.fs.realpathSync(paths.MAIL_DATA_DIR);
    if (!mailDataDir) throw new BoxError(BoxError.FS_ERROR, `Error resolving maildata: ${safe.error.message}`);

    const uploadConfig = {
        remotePath,
        backupTarget,
        dataLayout: new DataLayout(mailDataDir, []),
        progressTag: 'mail'
    };

    progressCallback({ message: 'Uploading mail snapshot' });

    const startTime = new Date();

    const { stats, integrity } = await runBackupUpload(uploadConfig, progressCallback);

    debug(`uploadMailSnapshot: took ${(new Date() - startTime)/1000} seconds`);

    await backupTargets.setSnapshotInfo(backupTarget, 'mail', { timestamp: new Date().toISOString() });

    return { stats, integrity };
}

async function backupMailWithTag(backupTarget, tag, options, progressCallback) {
    assert.strictEqual(typeof backupTarget, 'object');
    assert.strictEqual(typeof tag, 'string');
    assert.strictEqual(typeof options, 'object');
    assert.strictEqual(typeof progressCallback, 'function');

    debug(`backupMailWithTag: backing up mail with tag ${tag}`);

    const { stats, integrity } = await uploadMailSnapshot(backupTarget, progressCallback);

    const remotePath = addFileExtension(backupTarget, `${tag}/mail_v${constants.VERSION}`);

    debug(`backupMailWithTag: rotating to ${remotePath}`);

    const data = {
        remotePath,
        encryptionVersion: backupTarget.encryption ? 2 : null,
        packageVersion: constants.VERSION,
        type: backups.BACKUP_TYPE_MAIL,
        state: backups.BACKUP_STATE_CREATING,
        identifier: backups.BACKUP_IDENTIFIER_MAIL,
        dependsOn: [],
        manifest: null,
        preserveSecs: options.preserveSecs || 0,
        appConfig: null,
        targetId: backupTarget.id,
        stats,
        integrity
    };

    const id = await backups.add(data);
    const snapshotPath = addFileExtension(backupTarget, 'snapshot/mail');
    const [error] = await safe(copy(backupTarget, snapshotPath, remotePath, progressCallback));
    const state = error ? backups.BACKUP_STATE_ERROR : backups.BACKUP_STATE_NORMAL;
    await backups.setState(id, state);
    if (error) throw error;

    return id;
}

async function downloadMail(backupTarget, remotePath, progressCallback) {
    assert.strictEqual(typeof backupTarget, 'object');
    assert.strictEqual(typeof remotePath, 'string');
    assert.strictEqual(typeof progressCallback, 'function');

    const mailDataDir = safe.fs.realpathSync(paths.MAIL_DATA_DIR);
    if (!mailDataDir) throw new BoxError(BoxError.FS_ERROR, `Error resolving maildata: ${safe.error.message}`);
    const dataLayout = new DataLayout(mailDataDir, []);

    const startTime = new Date();

    await download(backupTarget, remotePath, dataLayout, progressCallback);
    debug('downloadMail: time: %s', (new Date() - startTime)/1000);
}

// this function is called from external process. calling process is expected to have a lock
async function fullBackup(backupTargetId, options, progressCallback) {
    assert.strictEqual(typeof backupTargetId, 'string');
    assert.strictEqual(typeof options, 'object');
    assert.strictEqual(typeof progressCallback, 'function');

    const backupTarget = await backupTargets.get(backupTargetId);
    if (!backupTarget) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Backup target not found');

    const tag = (new Date()).toISOString().replace(/[T.]/g, '-').replace(/[:Z]/g,''); // unique tag under which all apps/mail/box backs up

    const allApps = await apps.list();

    let percent = 1;
    const step = 100/(allApps.length+3);

    const appBackupIds = [];
    for (let i = 0; i < allApps.length; i++) {
        const app = allApps[i];
        percent += step;

        if (!app.enableBackup) {
            debug(`fullBackup: skipped backup ${app.fqdn} (${i+1}/${allApps.length}) since automatic backup disabled`);
            continue; // nothing to backup
        }

        progressCallback({ percent, message: `Backing up ${app.fqdn} (${i+1}/${allApps.length}). Waiting for lock` });
        await locks.wait(`${locks.TYPE_APP_BACKUP_PREFIX}${app.id}`);
        const startTime = new Date();
        const [appBackupError, appBackupId] = await safe(backupAppWithTag(app, backupTarget, tag, options, (progress) => progressCallback({ percent, message: progress.message })));
        debug(`fullBackup: app ${app.fqdn} backup finished. Took ${(new Date() - startTime)/1000} seconds`);
        await locks.release(`${locks.TYPE_APP_BACKUP_PREFIX}${app.id}`);
        if (appBackupError) throw appBackupError;
        if (appBackupId) appBackupIds.push(appBackupId); // backupId can be null if in BAD_STATE and never backed up
    }

    progressCallback({ percent, message: 'Backing up mail' });
    percent += step;
    const mailBackupId = await backupMailWithTag(backupTarget, tag, options, (progress) => progressCallback({ percent, message: progress.message }));

    progressCallback({ percent, message: 'Backing up system data' });
    percent += step;

    const dependsOn = appBackupIds.concat(mailBackupId);
    const backupId = await backupBox(backupTarget, dependsOn, tag, options, (progress) => progressCallback({ percent, message: progress.message }));
    return backupId;
}

// this function is called from external process
async function appBackup(appId, backupTargetId, options, progressCallback) {
    assert.strictEqual(typeof appId, 'string');
    assert.strictEqual(typeof backupTargetId, 'string');
    assert.strictEqual(typeof options, 'object');
    assert.strictEqual(typeof progressCallback, 'function');

    const app = await apps.get(appId);
    if (!app) throw new BoxError(BoxError.BAD_FIELD, 'App not found');

    const backupTarget = await backupTargets.get(backupTargetId);
    if (!backupTarget) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Backup target not found');

    await progressCallback({ percent: 1, message: `Backing up ${app.fqdn}. Waiting for lock` });
    const startTime = new Date();
    const backupId = await backupApp(app, backupTarget, options, progressCallback);
    await progressCallback({ percent: 100, message: `app ${app.fqdn} backup finished. Took ${(new Date() - startTime)/1000} seconds` });
    return backupId;
}