cloudron-box/src/test/backupcleaner-test.js

/* jslint node:true */
/* global it:false */
/* global describe:false */
/* global before:false */
/* global after:false */

'use strict';

const archives = require('../archives.js'),
    backupCleaner = require('../backupcleaner.js'),
    backups = require('../backups.js'),
    backupSites = require('../backupsites.js'),
    common = require('./common.js'),
    expect = require('expect.js'),
    moment = require('moment'),
    tasks = require('../tasks.js'),
    timers = require('timers/promises');

describe('backup cleaner', function () {
    const { setup, cleanup, app, getDefaultBackupSite, auditSource } = common;

    before(setup);
    after(cleanup);

    const backupTemplate = {
        id: null,
        remotePath: 'somepath',
        encryptionVersion: 2,
        packageVersion: '1.0.0',
        type: backups.BACKUP_TYPE_BOX,
        state: backups.BACKUP_STATE_NORMAL,
        identifier: 'box',
        dependsOn: [ 'dep1' ],
        manifest: null,
        preserveSecs: 0,
        appConfig: null
    };

    describe('retention', function () {
        it('keeps latest', function () {
            const backup = Object.assign({}, backupTemplate, { creationTime: moment().subtract(5, 's').toDate(), state: backups.BACKUP_STATE_NORMAL });
            backupCleaner._applyBackupRetention([backup], { keepWithinSecs: 1, keepLatest: true }, []);
            expect(backup.keepReason).to.be('latest');
        });

        it('does not keep latest', function () {
            const backup = { creationTime: moment().subtract(5, 's').toDate(), state: backups.BACKUP_STATE_NORMAL };
            backupCleaner._applyBackupRetention([backup], { keepWithinSecs: 1, keepLatest: false }, []);
            expect(backup.keepReason).to.be(undefined);
        });

        it('always keeps forever policy', function () {
            const backup = { creationTime: new Date() };
            backupCleaner._applyBackupRetention([backup], { keepWithinSecs: -1, keepLatest: true }, []);
            expect(backup.keepReason).to.be('keepWithinSecs');
        });

        it('preserveSecs takes precedence', function () {
            const backup = { creationTime: new Date(), preserveSecs: 3000 };
            backupCleaner._applyBackupRetention([backup], { keepWithinSecs: 1, keepLatest: true }, []);
            expect(backup.keepReason).to.be('preserveSecs');
        });

        it('1 daily', function () {
            const b = [
                { id: '0', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().toDate() },
                { id: '1', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(1, 'h').toDate() },
                { id: '2', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(3, 'h').toDate() },
                { id: '3', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(20, 'h').toDate() },
                { id: '4', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(5, 'd').toDate() }
            ];
            backupCleaner._applyBackupRetention(b, { keepDaily: 1, keepLatest: true }, []);
            expect(b[0].keepReason).to.be('keepDaily');
            expect(b[1].keepReason).to.be(undefined);
            expect(b[2].keepReason).to.be(undefined);
            expect(b[3].keepReason).to.be(undefined);
            expect(b[3].keepReason).to.be(undefined);
        });

        // if you are debugging this test, it's because of some timezone issue with all the hour substraction!
        it('2 daily, 1 weekly', function () {
            const b = [
                { id: '0', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().toDate() },
                { id: '1', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(1, 'h').toDate() },
                { id: '2', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(3, 'h').toDate() },
                { id: '3', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(26, 'h').toDate() },
                { id: '4', state: backups.BACKUP_STATE_ERROR, creationTime: moment().subtract(32, 'h').toDate() },
                { id: '5', state: backups.BACKUP_STATE_CREATING, creationTime: moment().subtract(50, 'h').toDate() },
                { id: '6', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(5, 'd').toDate() }
            ];
            backupCleaner._applyBackupRetention(b, { keepDaily: 2, keepWeekly: 1, keepLatest: false }, []);
            expect(b[0].keepReason).to.be('keepDaily'); // today
            expect(b[1].keepReason).to.be('keepWeekly'); // today
            expect(b[2].keepReason).to.be(undefined);
            expect(b[3].keepReason).to.be('keepDaily'); // yesterday
            expect(b[4].discardReason).to.be('error'); // errored
            expect(b[5].discardReason).to.be('creating-too-long'); // creating for too long
            expect(b[6].keepReason).to.be(undefined); // outside retention policy
        });

        it('2 daily, 3 monthly, 1 yearly', function () {
            const b = [
                { id: '0', state: backups.BACKUP_STATE_CREATING, creationTime: moment().toDate() },
                { id: '1', state: backups.BACKUP_STATE_ERROR, creationTime: moment().toDate() },
                { id: '2', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().toDate() },
                { id: '3', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(1, 'h').toDate() },
                { id: '4', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(3, 'h').toDate() },
                { id: '5', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(26, 'h').toDate() },
                { id: '6', state: backups.BACKUP_STATE_CREATING, creationTime: moment().subtract(49, 'h').toDate() },
                { id: '7', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(51, 'd').toDate() },
                { id: '8', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(84, 'd').toDate() },
                { id: '9', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(97, 'd').toDate() },
            ];
            backupCleaner._applyBackupRetention(b, { keepDaily: 2, keepMonthly: 3, keepYearly: 1, keepLatest: true }, []);
            expect(b[0].keepReason).to.be('creating');
            expect(b[1].discardReason).to.be('error'); // errored
            expect(b[2].keepReason).to.be('keepDaily');
            expect(b[3].keepReason).to.be('keepMonthly');
            expect(b[4].keepReason).to.be('keepYearly');
            expect(b[5].keepReason).to.be('keepDaily'); // yesterday
            expect(b[6].discardReason).to.be('creating-too-long'); // errored
            expect(b[7].keepReason).to.be('keepMonthly');
            expect(b[8].keepReason).to.be('keepMonthly');
            expect(b[9].keepReason).to.be(undefined);
        });
    });

    describe('task', function () {
        let site;

        const BACKUP_0_BOX = {
            id: null,
            remotePath: 'backup-box-0',
            identifier: 'box',
            encryptionVersion: null,
            packageVersion: '1.0.0',
            type: backups.BACKUP_TYPE_BOX,
            state: backups.BACKUP_STATE_NORMAL,
            dependsOn: [ 'backup-app-00', 'backup-app-01' ],
            manifest: null,
            preserveSecs: 0,
            appConfig: null,
            siteId: null
        };

        const BACKUP_0_APP_0 = { // backup of installed app
            id: null,
            remotePath: 'backup-app-00',
            identifier: app.id,
            encryptionVersion: null,
            packageVersion: '1.0.0',
            type: backups.BACKUP_TYPE_APP,
            state: backups.BACKUP_STATE_NORMAL,
            dependsOn: [],
            manifest: null,
            preserveSecs: 0,
            appConfig: null,
            siteId: null
        };

        const BACKUP_0_APP_1 = { // this app is uninstalled
            id: null,
            remotePath: 'backup-app-01',
            identifier: 'app1',
            encryptionVersion: null,
            packageVersion: '1.0.0',
            type: backups.BACKUP_TYPE_APP,
            state: backups.BACKUP_STATE_NORMAL,
            dependsOn: [],
            manifest: null,
            preserveSecs: 0,
            appConfig: null,
            siteId: null
        };

        const BACKUP_1_BOX = {
            id: null,
            remotePath: 'backup-box-1',
            encryptionVersion: null,
            packageVersion: '1.0.0',
            type: backups.BACKUP_TYPE_BOX,
            state: backups.BACKUP_STATE_NORMAL,
            identifier: 'box',
            dependsOn: [ 'backup-app-10', 'backup-app-11' ],
            manifest: null,
            preserveSecs: 0,
            appConfig: null,
            siteId: null
        };

        const BACKUP_1_APP_0 = {
            id: null,
            remotePath: 'backup-app-10',
            encryptionVersion: null,
            packageVersion: '1.0.0',
            type: backups.BACKUP_TYPE_APP,
            state: backups.BACKUP_STATE_NORMAL,
            identifier: app.id,
            dependsOn: [],
            manifest: null,
            preserveSecs: 0,
            appConfig: null,
            siteId: null
        };

        const BACKUP_1_APP_1 = {
            id: null,
            remotePath: 'backup-app-11',
            encryptionVersion: null,
            packageVersion: '1.0.0',
            type: backups.BACKUP_TYPE_APP,
            state: backups.BACKUP_STATE_NORMAL,
            identifier: 'app1',
            dependsOn: [],
            manifest: null,
            preserveSecs: 0,
            appConfig: null,
            siteId: null
        };

        const BACKUP_2_APP_2 = { // this is archived and left alone
            id: null,
            remotePath: 'backup-app-2',
            encryptionVersion: null,
            packageVersion: '2.0.0',
            type: backups.BACKUP_TYPE_APP,
            state: backups.BACKUP_STATE_NORMAL,
            identifier: 'app2',
            dependsOn: [],
            manifest: null,
            preserveSecs: 0,
            appConfig: null,
            siteId: null
        };

        before(async function () {
            site = await getDefaultBackupSite();
            await backupSites.setConfig(site, {
                provider: 'filesystem',
                backupDir: '/tmp/someplace',
            }, auditSource);
            await backupSites.setRetention(site, { keepWithinSecs: 1 }, auditSource);
            await backupSites.setSchedule(site, '00 00 23 * * *', auditSource);
        });

        async function cleanupBackups(site) {
            const taskId = await backupSites.startCleanupTask(site, auditSource);

            console.log('started task', taskId);

            while (true) {
                await timers.setTimeout(1000);

                const p = await tasks.get(taskId);

                if (p.percent !== 100) continue;
                if (p.error) throw new Error(`backup failed: ${p.error.message}`);

                return;
            }
        }

        it('succeeds without backups', async function () {
            await cleanupBackups(site);
        });

        it('add the backups', async function () {
            for (const b of [BACKUP_0_APP_0, BACKUP_0_APP_1, BACKUP_0_BOX, BACKUP_1_APP_0, BACKUP_1_APP_1, BACKUP_1_BOX, BACKUP_2_APP_2]) {
                b.siteId = site.id;
            }

            BACKUP_0_APP_0.id = await backups.add(BACKUP_0_APP_0);
            BACKUP_0_APP_1.id = await backups.add(BACKUP_0_APP_1);
            BACKUP_0_BOX.dependsOn = [ BACKUP_0_APP_0.id, BACKUP_0_APP_1.id ];
            BACKUP_0_BOX.id = await backups.add(BACKUP_0_BOX);

            await timers.setTimeout(2000); // space out backups

            BACKUP_1_APP_0.id = await backups.add(BACKUP_1_APP_0);
            BACKUP_1_APP_1.id = await backups.add(BACKUP_1_APP_1);
            BACKUP_1_BOX.dependsOn = [ BACKUP_1_APP_0.id, BACKUP_1_APP_1.id ];
            BACKUP_1_BOX.id = await backups.add(BACKUP_1_BOX);

            BACKUP_2_APP_2.id = await backups.add(BACKUP_2_APP_2);
            await archives.add(BACKUP_2_APP_2.id, {}, common.auditSource);
        });

        it('succeeds with box backups, keeps latest', async function () {
            await cleanupBackups(site);

            const results = await backups.listByTypePaged(backups.BACKUP_TYPE_BOX, 1, 1000);
            expect(results.length).to.equal(1);
            expect(results[0].id).to.equal(BACKUP_1_BOX.id);

            // check that app backups are gone as well. only backup_1 will remain
            const result = await backups.get(BACKUP_0_APP_0.id);
            expect(result).to.be(null);
        });

        it('does not remove expired backups if only one left', async function () {
            await cleanupBackups(site);

            const results = await backups.listByTypePaged(backups.BACKUP_TYPE_BOX, 1, 1000);
            expect(results[0].id).to.equal(BACKUP_1_BOX.id);

            // check that app backups are also still there. backup_1 is still there
            const result = await backups.get(BACKUP_1_APP_0.id);
            expect(result.id).to.equal(BACKUP_1_APP_0.id);
        });

        it('succeeds for app backups not referenced by a box backup', async function () {
            // add two dangling app backups not referenced by box backup. app1 is uninstalled. app0 is there
            for (const backup of [BACKUP_0_APP_0, BACKUP_0_APP_1]) {
                backup.id = await backups.add(backup);
            }

            await timers.setTimeout(2000); // wait for expiration

            await cleanupBackups(site);

            let result = await backups.listByTypePaged(backups.BACKUP_TYPE_APP, 1, 1000);
            expect(result.length).to.equal(4);
            result = result.sort((r1, r2) => r1.remotePath.localeCompare(r2.remotePath));
            expect(result[0].id).to.be(BACKUP_0_APP_0.id); // because app is installed, latest backup is preserved
            expect(result[1].id).to.be(BACKUP_1_APP_0.id); // referenced by box
            expect(result[2].id).to.be(BACKUP_1_APP_1.id); // referenced by box
            expect(result[3].id).to.be(BACKUP_2_APP_2.id); // referenced by archive
        });
    });
});