cloudron-box/src/backuptask.js

'use strict';

exports = module.exports = {
    fullBackup,

    restore,

    backupApp,
    downloadApp,

    backupMail,
    downloadMail,

    upload,
};

const apps = require('./apps.js'),
    assert = require('assert'),
    backupFormat = require('./backupformat.js'),
    backups = require('./backups.js'),
    BoxError = require('./boxerror.js'),
    constants = require('./constants.js'),
    DataLayout = require('./datalayout.js'),
    database = require('./database.js'),
    debug = require('debug')('box:backuptask'),
    df = require('./df.js'),
    locks = require('./locks.js'),
    path = require('path'),
    paths = require('./paths.js'),
    safe = require('safetydance'),
    services = require('./services.js'),
    shell = require('./shell.js')('backuptask'),
    storage = require('./storage.js');

const BACKUP_UPLOAD_CMD = path.join(__dirname, 'scripts/backupupload.js');

function canBackupApp(app) {
    // only backup apps that are installed or specific pending states

    // stopped apps cannot be backed up because addons might be down (redis)
    if (app.runState === apps.RSTATE_STOPPED) return false;

    // we used to check the health here but that doesn't work for stopped apps. it's better to just fail
    // and inform the user if the backup fails and the app addons have not been setup yet.
    return  app.installationState === apps.ISTATE_INSTALLED ||
            app.installationState === apps.ISTATE_PENDING_CONFIGURE ||
            app.installationState === apps.ISTATE_PENDING_BACKUP ||  // called from apptask
            app.installationState === apps.ISTATE_PENDING_UPDATE; // called from apptask
}

async function checkPreconditions(backupConfig, dataLayout) {
    assert.strictEqual(typeof backupConfig, 'object');
    assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');

    // check mount status before uploading
    const status = await backups.ensureMounted();
    debug(`checkPreconditions: mount point status is ${JSON.stringify(status)}`);
    if (status.state !== 'active') throw new BoxError(BoxError.MOUNT_ERROR, `Backup endpoint is not active: ${status.message}`);

    // check availabe size. this requires root for df to work
    const available = await storage.api(backupConfig.provider).getAvailableSize(backupConfig);
    let used = 0;
    for (const localPath of dataLayout.localPaths()) {
        debug(`checkPreconditions: getting disk usage of ${localPath}`);
        // du can error when files go missing as it is computing the size. it still prints some size anyway
        // to match df output in getAvailableSize() we must use disk usage size here and not apparent size
        const [duError, result] = await safe(shell.spawn('du', [ '--dereference-args', '--summarize', '--block-size=1', '--exclude=*.lock', '--exclude=dovecot.list.index.log.*', localPath], { encoding: 'utf8' }));
        if (duError) debug(`checkPreconditions: du error for ${localPath}. code: ${duError.code} stderror: ${duError.stderr}`);
        used += parseInt(duError ? duError.stdout : result, 10);
    }

    debug(`checkPreconditions: total required=${used} available=${available}`);

    const needed = 0.6 * used + (1024 * 1024 * 1024); // check if there is atleast 1GB left afterwards. aim for 60% because rsync/tgz won't need full 100%
    if (available <= needed) throw new BoxError(BoxError.FS_ERROR, `Not enough disk space for backup. Needed: ${df.prettyBytes(needed)} Available: ${df.prettyBytes(available)}`);
}

// this function is called via backupupload (since it needs root to traverse app's directory)
async function upload(remotePath, format, dataLayoutString, progressCallback) {
    assert.strictEqual(typeof remotePath, 'string');
    assert.strictEqual(typeof format, 'string');
    assert.strictEqual(typeof dataLayoutString, 'string');
    assert.strictEqual(typeof progressCallback, 'function');

    debug(`upload: path ${remotePath} format ${format} dataLayout ${dataLayoutString}`);

    const dataLayout = DataLayout.fromString(dataLayoutString);
    const backupConfig = await backups.getConfig();

    await checkPreconditions(backupConfig, dataLayout);

    await backupFormat.api(format).upload(backupConfig, remotePath, dataLayout, progressCallback);
}

async function download(backupConfig, remotePath, format, dataLayout, progressCallback) {
    assert.strictEqual(typeof backupConfig, 'object');
    assert.strictEqual(typeof remotePath, 'string');
    assert.strictEqual(typeof format, 'string');
    assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
    assert.strictEqual(typeof progressCallback, 'function');

    debug(`download: Downloading ${remotePath} of format ${format} to ${dataLayout.toString()}`);

    await backupFormat.api(format).download(backupConfig, remotePath, dataLayout, progressCallback);
}

async function restore(backupConfig, remotePath, progressCallback) {
    assert.strictEqual(typeof backupConfig, 'object');
    assert.strictEqual(typeof remotePath, 'string');
    assert.strictEqual(typeof progressCallback, 'function');

    const boxDataDir = safe.fs.realpathSync(paths.BOX_DATA_DIR);
    if (!boxDataDir) throw new BoxError(BoxError.FS_ERROR, `Error resolving boxdata: ${safe.error.message}`);
    const dataLayout = new DataLayout(boxDataDir, []);

    await download(backupConfig, remotePath, backupConfig.format, dataLayout, progressCallback);

    debug('restore: download completed, importing database');

    await database.importFromFile(`${dataLayout.localRoot()}/box.mysqldump`);
    debug('restore: database imported');

    await locks.releaseAll();
}

async function downloadApp(app, restoreConfig, progressCallback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof restoreConfig, 'object');
    assert.strictEqual(typeof progressCallback, 'function');

    const appDataDir = safe.fs.realpathSync(path.join(paths.APPS_DATA_DIR, app.id));
    if (!appDataDir) throw new BoxError(BoxError.FS_ERROR, safe.error.message);
    const dataLayout = new DataLayout(appDataDir, app.storageVolumeId ? [{ localDir: await apps.getStorageDir(app), remoteDir: 'data' }] : []);

    const startTime = new Date();
    const backupConfig = restoreConfig.backupConfig || await backups.getConfig();

    await download(backupConfig, restoreConfig.remotePath, restoreConfig.backupFormat, dataLayout, progressCallback);
    debug('downloadApp: time: %s', (new Date() - startTime)/1000);
}

async function runBackupUpload(uploadConfig, progressCallback) {
    assert.strictEqual(typeof uploadConfig, 'object');
    assert.strictEqual(typeof progressCallback, 'function');

    const { remotePath, backupConfig, dataLayout, progressTag } = uploadConfig;
    assert.strictEqual(typeof remotePath, 'string');
    assert.strictEqual(typeof backupConfig, 'object');
    assert.strictEqual(typeof progressTag, 'string');
    assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');

    // https://stackoverflow.com/questions/48387040/node-js-recommended-max-old-space-size
    const envCopy = Object.assign({}, process.env);
    if (backupConfig.limits?.memoryLimit >= 2*1024*1024*1024) {
        const heapSize = Math.min((backupConfig.limits.memoryLimit/1024/1024) - 256, 8192);
        debug(`runBackupUpload: adjusting heap size to ${heapSize}M`);
        envCopy.NODE_OPTIONS = `--max-old-space-size=${heapSize}`;
    }

    let result = ''; // the script communicates error result as a string
    function onMessage(progress) { // this is { message } or { result }
        if ('message' in progress) return progressCallback({ message: `${progress.message} (${progressTag})` });
        debug(`runBackupUpload: result - ${JSON.stringify(progress)}`);
        result = progress.result;
    }

    const [error] = await safe(shell.promises.sudo([ BACKUP_UPLOAD_CMD, remotePath, backupConfig.format, dataLayout.toString() ], { env: envCopy, preserveEnv: true, ipc: true, onMessage, outputHasTimestamps: true }));
    if (error && (error.code === null /* signal */ || (error.code !== 0 && error.code !== 50))) { // backuptask crashed
        debug(`runBackupUpload: backuptask crashed`, error);
        throw new BoxError(BoxError.INTERNAL_ERROR, 'Backuptask crashed');
    } else if (error && error.code === 50) { // exited with error
        throw new BoxError(BoxError.EXTERNAL_ERROR, result);
    }
}

async function snapshotBox(progressCallback) {
    assert.strictEqual(typeof progressCallback, 'function');

    progressCallback({ message: 'Snapshotting box' });

    const startTime = new Date();

    await database.exportToFile(`${paths.BOX_DATA_DIR}/box.mysqldump`);
    debug(`snapshotBox: took ${(new Date() - startTime)/1000} seconds`);
}

async function uploadBoxSnapshot(backupConfig, progressCallback) {
    assert.strictEqual(typeof backupConfig, 'object');
    assert.strictEqual(typeof progressCallback, 'function');

    await snapshotBox(progressCallback);

    const boxDataDir = safe.fs.realpathSync(paths.BOX_DATA_DIR);
    if (!boxDataDir) throw new BoxError(BoxError.FS_ERROR, `Error resolving boxdata: ${safe.error.message}`);

    const uploadConfig = {
        remotePath: 'snapshot/box',
        backupConfig,
        dataLayout: new DataLayout(boxDataDir, []),
        progressTag: 'box'
    };

    progressCallback({ message: 'Uploading box snapshot' });

    const startTime = new Date();

    await runBackupUpload(uploadConfig, progressCallback);

    debug(`uploadBoxSnapshot: took ${(new Date() - startTime)/1000} seconds`);

    await backups.setSnapshotInfo('box', { timestamp: new Date().toISOString(), format: backupConfig.format });
}

async function copy(backupConfig, srcRemotePath, destRemotePath, progressCallback) {
    assert.strictEqual(typeof backupConfig, 'object');
    assert.strictEqual(typeof srcRemotePath, 'string');
    assert.strictEqual(typeof destRemotePath, 'string');
    assert.strictEqual(typeof progressCallback, 'function');

    const { provider, format } = backupConfig;
    const oldFilePath = backupFormat.api(format).getBackupFilePath(backupConfig, srcRemotePath);
    const newFilePath = backupFormat.api(format).getBackupFilePath(backupConfig, destRemotePath);

    const startTime = new Date();
    const [copyError] = await safe(storage.api(provider).copy(backupConfig, oldFilePath, newFilePath, progressCallback));
    if (copyError) {
        debug(`copy: copied to ${destRemotePath} errored. error: ${copyError.message}`);
        throw copyError;
    }
    debug(`copy: copied successfully to ${destRemotePath}. Took ${(new Date() - startTime)/1000} seconds`);
}

async function rotateBoxBackup(backupConfig, tag, options, dependsOn, progressCallback) {
    assert.strictEqual(typeof backupConfig, 'object');
    assert.strictEqual(typeof tag, 'string');
    assert.strictEqual(typeof options, 'object');
    assert(Array.isArray(dependsOn));
    assert.strictEqual(typeof progressCallback, 'function');

    const remotePath = `${tag}/box_v${constants.VERSION}`;
    const format = backupConfig.format;

    debug(`rotateBoxBackup: rotating to id ${remotePath}`);

    const data = {
        remotePath,
        encryptionVersion: backupConfig.encryption ? 2 : null,
        packageVersion: constants.VERSION,
        type: backups.BACKUP_TYPE_BOX,
        state: backups.BACKUP_STATE_CREATING,
        identifier: backups.BACKUP_IDENTIFIER_BOX,
        dependsOn,
        manifest: null,
        format,
        preserveSecs: options.preserveSecs || 0,
        appConfig: null
    };

    const id = await backups.add(data);
    const [error] = await safe(copy(backupConfig, 'snapshot/box', remotePath, progressCallback));
    const state = error ? backups.BACKUP_STATE_ERROR : backups.BACKUP_STATE_NORMAL;
    await backups.setState(id, state);
    if (error) throw error;

    return id;
}

async function backupBox(dependsOn, tag, options, progressCallback) {
    assert(Array.isArray(dependsOn));
    assert.strictEqual(typeof tag, 'string');
    assert.strictEqual(typeof options, 'object');
    assert.strictEqual(typeof progressCallback, 'function');

    const backupConfig = await backups.getConfig();

    await uploadBoxSnapshot(backupConfig, progressCallback);
    return await rotateBoxBackup(backupConfig, tag, options, dependsOn, progressCallback);
}

async function rotateAppBackup(backupConfig, app, tag, options, progressCallback) {
    assert.strictEqual(typeof backupConfig, 'object');
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof tag, 'string');
    assert.strictEqual(typeof options, 'object');
    assert.strictEqual(typeof progressCallback, 'function');

    const snapshotInfo = backups.getSnapshotInfo(app.id);

    const manifest = snapshotInfo.restoreConfig ? snapshotInfo.restoreConfig.manifest : snapshotInfo.manifest; // compat
    const remotePath = `${tag}/app_${app.fqdn}_v${manifest.version}`;
    const format = backupConfig.format;

    debug(`rotateAppBackup: rotating ${app.fqdn} to path ${remotePath}`);

    const data = {
        remotePath,
        encryptionVersion: backupConfig.encryption ? 2 : null,
        packageVersion: manifest.version,
        type: backups.BACKUP_TYPE_APP,
        state: backups.BACKUP_STATE_CREATING,
        identifier: app.id,
        dependsOn: [],
        manifest,
        format,
        preserveSecs: options.preserveSecs || 0,
        appConfig: app
    };

    const id = await backups.add(data);
    const [error] = await safe(copy(backupConfig,  `snapshot/app_${app.id}`, remotePath, progressCallback));
    const state = error ? backups.BACKUP_STATE_ERROR : backups.BACKUP_STATE_NORMAL;
    await backups.setState(id, state);
    if (error) throw error;

    return id;
}

async function backupApp(app, options, progressCallback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof options, 'object');
    assert.strictEqual(typeof progressCallback, 'function');

    if (options.snapshotOnly) return await snapshotApp(app, progressCallback);

    const tag = (new Date()).toISOString().replace(/[T.]/g, '-').replace(/[:Z]/g,'');

    debug(`backupApp: backing up ${app.fqdn} with tag ${tag}`);

    return await backupAppWithTag(app, tag, options, progressCallback);
}

async function snapshotApp(app, progressCallback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof progressCallback, 'function');

    const startTime = new Date();
    progressCallback({ message: `Snapshotting app ${app.fqdn}` });

    await apps.writeConfig(app);
    await services.backupAddons(app, app.manifest.addons);

    debug(`snapshotApp: ${app.fqdn} took ${(new Date() - startTime)/1000} seconds`);
}

async function uploadAppSnapshot(backupConfig, app, progressCallback) {
    assert.strictEqual(typeof backupConfig, 'object');
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof progressCallback, 'function');

    await snapshotApp(app, progressCallback);

    const remotePath = `snapshot/app_${app.id}`;
    const appDataDir = safe.fs.realpathSync(path.join(paths.APPS_DATA_DIR, app.id));
    if (!appDataDir) throw new BoxError(BoxError.FS_ERROR, `Error resolving appsdata: ${safe.error.message}`);

    const dataLayout = new DataLayout(appDataDir, app.storageVolumeId ? [{ localDir: await apps.getStorageDir(app), remoteDir: 'data' }] : []);

    progressCallback({ message: `Uploading app snapshot ${app.fqdn}`});

    const uploadConfig = {
        remotePath,
        backupConfig,
        dataLayout,
        progressTag: app.fqdn
    };

    const startTime = new Date();

    await runBackupUpload(uploadConfig, progressCallback);

    debug(`uploadAppSnapshot: ${app.fqdn} uploaded to ${remotePath}. ${(new Date() - startTime)/1000} seconds`);

    await backups.setSnapshotInfo(app.id, { timestamp: new Date().toISOString(), manifest: app.manifest, format: backupConfig.format });
}

async function backupAppWithTag(app, tag, options, progressCallback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof tag, 'string');
    assert.strictEqual(typeof options, 'object');
    assert.strictEqual(typeof progressCallback, 'function');

    if (!canBackupApp(app)) { // if we cannot backup, reuse it's most recent backup
        const results = await backups.getByIdentifierAndStatePaged(app.id, backups.BACKUP_STATE_NORMAL, 1, 1);
        if (results.length === 0) return null; // no backup to re-use

        return results[0].id;
    }

    const backupConfig = await backups.getConfig();

    await uploadAppSnapshot(backupConfig, app, progressCallback);
    return await rotateAppBackup(backupConfig, app, tag, options, progressCallback);
}

async function uploadMailSnapshot(backupConfig, progressCallback) {
    assert.strictEqual(typeof backupConfig, 'object');
    assert.strictEqual(typeof progressCallback, 'function');

    const mailDataDir = safe.fs.realpathSync(paths.MAIL_DATA_DIR);
    if (!mailDataDir) throw new BoxError(BoxError.FS_ERROR, `Error resolving maildata: ${safe.error.message}`);

    const uploadConfig = {
        remotePath: 'snapshot/mail',
        backupConfig,
        dataLayout: new DataLayout(mailDataDir, []),
        progressTag: 'mail'
    };

    progressCallback({ message: 'Uploading mail snapshot' });

    const startTime = new Date();

    await runBackupUpload(uploadConfig, progressCallback);

    debug(`uploadMailSnapshot: took ${(new Date() - startTime)/1000} seconds`);

    await backups.setSnapshotInfo('mail', { timestamp: new Date().toISOString(), format: backupConfig.format });
}

async function rotateMailBackup(backupConfig, tag, options, progressCallback) {
    assert.strictEqual(typeof backupConfig, 'object');
    assert.strictEqual(typeof tag, 'string');
    assert.strictEqual(typeof options, 'object');
    assert.strictEqual(typeof progressCallback, 'function');

    const remotePath = `${tag}/mail_v${constants.VERSION}`;
    const format = backupConfig.format;

    debug(`rotateMailBackup: rotating to ${remotePath}`);

    const data = {
        remotePath,
        encryptionVersion: backupConfig.encryption ? 2 : null,
        packageVersion: constants.VERSION,
        type: backups.BACKUP_TYPE_MAIL,
        state: backups.BACKUP_STATE_CREATING,
        identifier: backups.BACKUP_IDENTIFIER_MAIL,
        dependsOn: [],
        manifest: null,
        format,
        preserveSecs: options.preserveSecs || 0,
        appConfig: null
    };

    const id = await backups.add(data);
    const [error] = await safe(copy(backupConfig, 'snapshot/mail', remotePath, progressCallback));
    const state = error ? backups.BACKUP_STATE_ERROR : backups.BACKUP_STATE_NORMAL;
    await backups.setState(id, state);
    if (error) throw error;

    return id;
}

async function backupMailWithTag(tag, options, progressCallback) {
    assert.strictEqual(typeof tag, 'string');
    assert.strictEqual(typeof options, 'object');
    assert.strictEqual(typeof progressCallback, 'function');

    debug(`backupMailWithTag: backing up mail with tag ${tag}`);

    const backupConfig = await backups.getConfig();
    await uploadMailSnapshot(backupConfig, progressCallback);
    return await rotateMailBackup(backupConfig, tag, options, progressCallback);
}

async function backupMail(options, progressCallback) {
    assert.strictEqual(typeof options, 'object');
    assert.strictEqual(typeof progressCallback, 'function');

    const tag = (new Date()).toISOString().replace(/[T.]/g, '-').replace(/[:Z]/g,'');
    debug(`backupMail: backing up mail with tag ${tag}`);
    return await backupMailWithTag(tag, options, progressCallback);
}

async function downloadMail(restoreConfig, progressCallback) {
    assert.strictEqual(typeof restoreConfig, 'object');
    assert.strictEqual(typeof progressCallback, 'function');

    const mailDataDir = safe.fs.realpathSync(paths.MAIL_DATA_DIR);
    if (!mailDataDir) throw new BoxError(BoxError.FS_ERROR, `Error resolving maildata: ${safe.error.message}`);
    const dataLayout = new DataLayout(mailDataDir, []);

    const startTime = new Date();

    await download(restoreConfig.backupConfig, restoreConfig.remotePath, restoreConfig.backupFormat, dataLayout, progressCallback);
    debug('downloadMail: time: %s', (new Date() - startTime)/1000);
}

// this function is called from external process. calling process is expected to have a lock
async function fullBackup(options, progressCallback) {
    assert.strictEqual(typeof options, 'object');
    assert.strictEqual(typeof progressCallback, 'function');

    const tag = (new Date()).toISOString().replace(/[T.]/g, '-').replace(/[:Z]/g,''); // unique tag under which all apps/mail/box backs up

    const allApps = await apps.list();

    let percent = 1;
    const step = 100/(allApps.length+3);

    const appBackupIds = [];
    for (let i = 0; i < allApps.length; i++) {
        const app = allApps[i];
        percent += step;

        if (!app.enableBackup) {
            debug(`fullBackup: skipped backup ${app.fqdn} (${i+1}/${allApps.length}) since automatic backup disabled`);
            continue; // nothing to backup
        }

        progressCallback({ percent, message: `Backing up ${app.fqdn} (${i+1}/${allApps.length}). Waiting for lock` });
        await locks.wait(`${locks.TYPE_APP_PREFIX}${app.id}`);
        const startTime = new Date();
        const appBackupId = await backupAppWithTag(app, tag, options, (progress) => progressCallback({ percent, message: progress.message }));
        debug(`fullBackup: app ${app.fqdn} backup finished. Took ${(new Date() - startTime)/1000} seconds`);
        await locks.release(`${locks.TYPE_APP_PREFIX}${app.id}`);
        if (appBackupId) appBackupIds.push(appBackupId); // backupId can be null if in BAD_STATE and never backed up
    }

    progressCallback({ percent, message: 'Backing up mail' });
    percent += step;
    const mailBackupId = await backupMailWithTag(tag, options, (progress) => progressCallback({ percent, message: progress.message }));

    progressCallback({ percent, message: 'Backing up system data' });
    percent += step;

    const dependsOn = appBackupIds.concat(mailBackupId);
    const backupId = await backupBox(dependsOn, tag, options, (progress) => progressCallback({ percent, message: progress.message }));
    return backupId;
}