cloudron-box/src/backupcleaner.js

'use strict';

exports = module.exports = {
    run,

    _applyBackupRetentionPolicy: applyBackupRetentionPolicy
};

const apps = require('./apps.js'),
    assert = require('assert'),
    backups = require('./backups.js'),
    constants = require('./constants.js'),
    debug = require('debug')('box:backupcleaner'),
    moment = require('moment'),
    path = require('path'),
    paths = require('./paths.js'),
    safe = require('safetydance'),
    settings = require('./settings.js'),
    storage = require('./storage.js'),
    util = require('util'),
    _ = require('underscore');

function applyBackupRetentionPolicy(allBackups, policy, referencedBackupIds) {
    assert(Array.isArray(allBackups));
    assert.strictEqual(typeof policy, 'object');
    assert(Array.isArray(referencedBackupIds));

    const now = new Date();

    for (const backup of allBackups) {
        if (backup.state === backups.BACKUP_STATE_ERROR) {
            backup.discardReason = 'error';
        } else if (backup.state === backups.BACKUP_STATE_CREATING) {
            if ((now - backup.creationTime) < 48*60*60*1000) backup.keepReason = 'creating';
            else backup.discardReason = 'creating-too-long';
        } else if (referencedBackupIds.includes(backup.id)) {
            backup.keepReason = 'reference';
        } else if ((now - backup.creationTime) < (backup.preserveSecs * 1000)) {
            backup.keepReason = 'preserveSecs';
        } else if ((now - backup.creationTime < policy.keepWithinSecs * 1000) || policy.keepWithinSecs < 0) {
            backup.keepReason = 'keepWithinSecs';
        }
    }

    const KEEP_FORMATS = {
        keepDaily: 'Y-M-D',
        keepWeekly: 'Y-W',
        keepMonthly: 'Y-M',
        keepYearly: 'Y'
    };

    for (const format of [ 'keepDaily', 'keepWeekly', 'keepMonthly', 'keepYearly' ]) {
        if (!(format in policy)) continue;

        const n = policy[format]; // we want to keep "n" backups of format
        if (!n) continue; // disabled rule

        let lastPeriod = null, keptSoFar = 0;
        for (const backup of allBackups) {
            if (backup.discardReason) continue; // already discarded for some reason
            if (backup.keepReason && backup.keepReason !== 'reference') continue; // kept for some other reason
            const period = moment(backup.creationTime).format(KEEP_FORMATS[format]);
            if (period === lastPeriod) continue; // already kept for this period

            lastPeriod = period;
            backup.keepReason = backup.keepReason ? `${backup.keepReason}+${format}` : format;
            if (++keptSoFar === n) break;
        }
    }

    if (policy.keepLatest) {
        let latestNormalBackup = allBackups.find(b => b.state === backups.BACKUP_STATE_NORMAL);
        if (latestNormalBackup && !latestNormalBackup.keepReason) latestNormalBackup.keepReason = 'latest';
    }

    for (const backup of allBackups) {
        debug(`applyBackupRetentionPolicy: ${backup.id} ${backup.type} ${backup.keepReason || backup.discardReason || 'unprocessed'}`);
    }
}

async function cleanupBackup(backupConfig, backup, progressCallback) {
    assert.strictEqual(typeof backupConfig, 'object');
    assert.strictEqual(typeof backup, 'object');
    assert.strictEqual(typeof progressCallback, 'function');

    const backupFilePath = storage.getBackupFilePath(backupConfig, backup.id, backup.format);

    return new Promise((resolve) => {
        function done(error) {
            if (error) {
                debug('cleanupBackup: error removing backup %j : %s', backup, error.message);
                return resolve();
            }

            // prune empty directory if possible
            storage.api(backupConfig.provider).remove(backupConfig, path.dirname(backupFilePath), async function (error) {
                if (error) debug('cleanupBackup: unable to prune backup directory %s : %s', path.dirname(backupFilePath), error.message);

                const [delError] = await safe(backups.del(backup.id));
                if (delError) debug('cleanupBackup: error removing from database', delError);
                else debug('cleanupBackup: removed %s', backup.id);

                resolve();
            });
        }

        if (backup.format ==='tgz') {
            progressCallback({ message: `${backup.id}: Removing ${backupFilePath}`});
            storage.api(backupConfig.provider).remove(backupConfig, backupFilePath, done);
        } else {
            const events = storage.api(backupConfig.provider).removeDir(backupConfig, backupFilePath);
            events.on('progress', (message) => progressCallback({ message: `${backup.id}: ${message}` }));
            events.on('done', done);
        }
    });
}

async function cleanupAppBackups(backupConfig, referencedAppBackupIds, progressCallback) {
    assert.strictEqual(typeof backupConfig, 'object');
    assert(Array.isArray(referencedAppBackupIds));
    assert.strictEqual(typeof progressCallback, 'function');

    let removedAppBackupIds = [];

    const allApps = await apps.list();
    const allAppIds = allApps.map(a => a.id);

    const appBackups = await backups.getByTypePaged(backups.BACKUP_TYPE_APP, 1, 1000);

    // collate the backups by app id. note that the app could already have been uninstalled
    let appBackupsById = {};
    for (const appBackup of appBackups) {
        if (!appBackupsById[appBackup.identifier]) appBackupsById[appBackup.identifier] = [];
        appBackupsById[appBackup.identifier].push(appBackup);
    }

    // apply backup policy per app. keep latest backup only for existing apps
    let appBackupsToRemove = [];
    for (const appId of Object.keys(appBackupsById)) {
        applyBackupRetentionPolicy(appBackupsById[appId], _.extend({ keepLatest: allAppIds.includes(appId) }, backupConfig.retentionPolicy), referencedAppBackupIds);
        appBackupsToRemove = appBackupsToRemove.concat(appBackupsById[appId].filter(b => !b.keepReason));
    }

    for (const appBackup of appBackupsToRemove) {
        await progressCallback({ message: `Removing app backup (${appBackup.identifier}): ${appBackup.id}`});
        removedAppBackupIds.push(appBackup.id);
        await cleanupBackup(backupConfig, appBackup, progressCallback); // never errors
    }

    debug('cleanupAppBackups: done');

    return removedAppBackupIds;
}

async function cleanupMailBackups(backupConfig, referencedAppBackupIds, progressCallback) {
    assert.strictEqual(typeof backupConfig, 'object');
    assert(Array.isArray(referencedAppBackupIds));
    assert.strictEqual(typeof progressCallback, 'function');

    let removedMailBackupIds = [];

    const mailBackups = await backups.getByTypePaged(backups.BACKUP_TYPE_MAIL, 1, 1000);

    applyBackupRetentionPolicy(mailBackups, _.extend({ keepLatest: true }, backupConfig.retentionPolicy), referencedAppBackupIds);

    for (const mailBackup of mailBackups) {
        if (mailBackup.keepReason) continue;
        await progressCallback({ message: `Removing mail backup ${mailBackup.id}`});
        removedMailBackupIds.push(mailBackup.id);
        await cleanupBackup(backupConfig, mailBackup, progressCallback); // never errors
    }

    debug('cleanupMailBackups: done');

    return removedMailBackupIds;
}

async function cleanupBoxBackups(backupConfig, progressCallback) {
    assert.strictEqual(typeof backupConfig, 'object');
    assert.strictEqual(typeof progressCallback, 'function');

    let referencedAppBackupIds = [], removedBoxBackupIds = [];

    const boxBackups = await backups.getByTypePaged(backups.BACKUP_TYPE_BOX, 1, 1000);

    applyBackupRetentionPolicy(boxBackups, _.extend({ keepLatest: true }, backupConfig.retentionPolicy), [] /* references */);

    for (const boxBackup of boxBackups) {
        if (boxBackup.keepReason) {
            referencedAppBackupIds = referencedAppBackupIds.concat(boxBackup.dependsOn);
            continue;
        }

        await progressCallback({ message: `Removing box backup ${boxBackup.id}`});

        removedBoxBackupIds.push(boxBackup.id);
        await cleanupBackup(backupConfig, boxBackup, progressCallback);
    }

    debug('cleanupBoxBackups: done');

    return { removedBoxBackupIds, referencedAppBackupIds };
}

async function cleanupMissingBackups(backupConfig, progressCallback) {
    assert.strictEqual(typeof backupConfig, 'object');
    assert.strictEqual(typeof progressCallback, 'function');

    const perPage = 1000;
    let missingBackupIds = [];
    const backupExists = util.promisify(storage.api(backupConfig.provider).exists);

    if (constants.TEST) return missingBackupIds;

    let page = 1, result = [];
    do {
        result = await backups.list(page, perPage);

        for (const backup of result) {
            let backupFilePath = storage.getBackupFilePath(backupConfig, backup.id, backup.format);
            if (backup.format === 'rsync') backupFilePath = backupFilePath + '/'; // add trailing slash to indicate directory

            const [existsError, exists] = await safe(backupExists(backupConfig, backupFilePath));
            if (existsError || exists) continue;

            await progressCallback({ message: `Removing missing backup ${backup.id}`});

            const [delError] = await safe(backups.del(backup.id));
            if (delError) debug(`cleanupBackup: error removing ${backup.id} from database`, delError);

            missingBackupIds.push(backup.id);
        }

        ++ page;
    } while (result.length === perPage);

    debug('cleanupMissingBackups: done');

    return missingBackupIds;
}

// removes the snapshots of apps that have been uninstalled
async function cleanupSnapshots(backupConfig) {
    assert.strictEqual(typeof backupConfig, 'object');

    const contents = safe.fs.readFileSync(paths.SNAPSHOT_INFO_FILE, 'utf8');
    const info = safe.JSON.parse(contents);
    if (!info) return;

    delete info.box;

    for (const appId of Object.keys(info)) {
        const app = await apps.get(appId);
        if (app) continue; // app is still installed

        await new Promise((resolve) => {
            async function done(/* ignoredError */) {
                safe.fs.unlinkSync(path.join(paths.BACKUP_INFO_DIR, `${appId}.sync.cache`));
                safe.fs.unlinkSync(path.join(paths.BACKUP_INFO_DIR, `${appId}.sync.cache.new`));

                await safe(backups.setSnapshotInfo(appId, null /* info */), { debug });
                debug(`cleanupSnapshots: cleaned up snapshot of app ${appId}`);

                resolve();
            }

            if (info[appId].format ==='tgz') {
                storage.api(backupConfig.provider).remove(backupConfig, storage.getBackupFilePath(backupConfig, `snapshot/app_${appId}`, info[appId].format), done);
            } else {
                const events = storage.api(backupConfig.provider).removeDir(backupConfig, storage.getBackupFilePath(backupConfig, `snapshot/app_${appId}`, info[appId].format));
                events.on('progress', function (detail) { debug(`cleanupSnapshots: ${detail}`); });
                events.on('done', done);
            }
        });
    }

    debug('cleanupSnapshots: done');
}

async function run(progressCallback) {
    assert.strictEqual(typeof progressCallback, 'function');

    const backupConfig = await settings.getBackupConfig();

    if (backupConfig.retentionPolicy.keepWithinSecs < 0) {
        debug('cleanup: keeping all backups');
        return {};
    }

    await progressCallback({ percent: 10, message: 'Cleaning box backups' });
    const { removedBoxBackupIds, referencedAppBackupIds } = await cleanupBoxBackups(backupConfig, progressCallback);

    await progressCallback({ percent: 20, message: 'Cleaning mail backups' });
    const removedMailBackupIds = await cleanupMailBackups(backupConfig, referencedAppBackupIds, progressCallback);

    await progressCallback({ percent: 40, message: 'Cleaning app backups' });
    const removedAppBackupIds = await cleanupAppBackups(backupConfig, referencedAppBackupIds, progressCallback);

    await progressCallback({ percent: 70, message: 'Cleaning missing backups' });
    const missingBackupIds = await cleanupMissingBackups(backupConfig, progressCallback);

    await progressCallback({ percent: 90, message: 'Cleaning snapshots' });
    await cleanupSnapshots(backupConfig);

    return { removedBoxBackupIds, removedMailBackupIds, removedAppBackupIds, missingBackupIds };
}