diff --git a/src/apps.js b/src/apps.js index c1c6e3b49..504a3e513 100644 --- a/src/apps.js +++ b/src/apps.js @@ -151,6 +151,7 @@ exports = module.exports = { const appTaskManager = require('./apptaskmanager.js'), archives = require('./archives.js'), assert = require('assert'), + backupListing = require('./backuplisting.js'), backups = require('./backups.js'), BoxError = require('./boxerror.js'), constants = require('./constants.js'), @@ -2300,7 +2301,7 @@ async function restore(app, backupId, auditSource) { if (error) throw error; // for empty or null backupId, use existing manifest to mimic a reinstall - const backupInfo = backupId ? await backups.get(backupId) : { manifest: app.manifest }; + const backupInfo = backupId ? await backupListing.get(backupId) : { manifest: app.manifest }; if (!backupInfo) throw new BoxError(BoxError.BAD_FIELD, 'No such backup'); const manifest = backupInfo.manifest; @@ -2424,7 +2425,7 @@ async function clone(app, data, user, auditSource) { assert.strictEqual(typeof subdomain, 'string'); assert.strictEqual(typeof domain, 'string'); - const backupInfo = await backups.get(backupId); + const backupInfo = await backupListing.get(backupId); if (!backupInfo) throw new BoxError(BoxError.NOT_FOUND, 'Backup not found'); if (!backupInfo.manifest) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Could not detect restore manifest'); @@ -2503,7 +2504,7 @@ async function unarchive(archive, data, auditSource) { assert.strictEqual(typeof data, 'object'); assert(auditSource && typeof auditSource === 'object'); - const backup = await backups.get(archive.backupId); + const backup = await backupListing.get(archive.backupId); const restoreConfig = { remotePath: backup.remotePath, backupFormat: backup.format }; const subdomain = data.subdomain.toLowerCase(), @@ -2599,7 +2600,7 @@ async function archive(app, backupId, auditSource) { if (app.manifest.id === constants.PROXY_APP_APPSTORE_ID) throw new BoxError(BoxError.BAD_FIELD, 'cannot archive proxy app'); - const result = await backups.getByIdentifierAndStatePaged(app.id, backups.BACKUP_STATE_NORMAL, 1, 1); + const result = await backupListing.getByIdentifierAndStatePaged(app.id, backupListing.BACKUP_STATE_NORMAL, 1, 1); if (result.length === 0) throw new BoxError(BoxError.BAD_STATE, 'No recent backup to archive'); if (result[0].id !== backupId) throw new BoxError(BoxError.BAD_STATE, 'Latest backup id has changed'); @@ -2785,7 +2786,7 @@ async function backup(app, auditSource) { // background tasks.startTask(taskId, { timeout: 24 * 60 * 60 * 1000 /* 24 hours */, nice: 15, memoryLimit, oomScoreAdjust: -999 }) .then(async (backupId) => { - const backup = await backups.get(backupId); // if task crashed, no result + const backup = await backupListing.get(backupId); // if task crashed, no result await eventlog.add(eventlog.ACTION_APP_BACKUP_FINISH, auditSource, { app, success: !!backup, errorMessage: '', remotePath: backup?.remotePath, backupId: backupId }); }) .catch(async (error) => { @@ -2805,7 +2806,7 @@ async function listBackups(app, page, perPage) { assert(typeof page === 'number' && page > 0); assert(typeof perPage === 'number' && perPage > 0); - return await backups.getByIdentifierAndStatePaged(app.id, backups.BACKUP_STATE_NORMAL, page, perPage); + return await backupListing.getByIdentifierAndStatePaged(app.id, backupListing.BACKUP_STATE_NORMAL, page, perPage); } async function updateBackup(app, backupId, data) { @@ -2813,18 +2814,18 @@ async function updateBackup(app, backupId, data) { assert.strictEqual(typeof backupId, 'string'); assert.strictEqual(typeof data, 'object'); - const backup = await backups.get(backupId); + const backup = await backupListing.get(backupId); if (!backup) throw new BoxError(BoxError.NOT_FOUND, 'Backup not found'); if (backup.identifier !== app.id) throw new BoxError(BoxError.NOT_FOUND, 'Backup not found'); // some other app's backup - await backups.update(backupId, data); + await backupListing.update(backupId, data); } async function getBackupDownloadStream(app, backupId) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof backupId, 'string'); - const backup = await backups.get(backupId); + const backup = await backupListing.get(backupId); if (!backup) throw new BoxError(BoxError.NOT_FOUND, 'Backup not found'); if (backup.identifier !== app.id) throw new BoxError(BoxError.NOT_FOUND, 'Backup not found'); // some other app's backup if (backup.format !== 'tgz') throw new BoxError(BoxError.BAD_STATE, 'only tgz backups can be downloaded'); @@ -2856,7 +2857,7 @@ async function restoreApps(apps, options, auditSource) { apps = apps.filter(app => app.installationState !== exports.ISTATE_PENDING_RESTORE); // safeguard against tasks being created non-stop if we crash on startup for (const app of apps) { - const [error, result] = await safe(backups.getByIdentifierAndStatePaged(app.id, backups.BACKUP_STATE_NORMAL, 1, 1)); + const [error, result] = await safe(backupListing.getByIdentifierAndStatePaged(app.id, backupListing.BACKUP_STATE_NORMAL, 1, 1)); let installationState, restoreConfig, oldManifest; if (!error && result.length) { installationState = exports.ISTATE_PENDING_RESTORE; diff --git a/src/backupcleaner.js b/src/backupcleaner.js index 100ecc6c8..0a59782d2 100644 --- a/src/backupcleaner.js +++ b/src/backupcleaner.js @@ -12,6 +12,7 @@ const apps = require('./apps.js'), archives = require('./archives.js'), assert = require('assert'), backupFormat = require('./backupformat.js'), + backupListing = require('./backuplisting.js'), backups = require('./backups.js'), constants = require('./constants.js'), debug = require('debug')('box:backupcleaner'), @@ -29,9 +30,9 @@ function applyBackupRetention(allBackups, retention, referencedBackupIds) { const now = new Date(); for (const backup of allBackups) { - if (backup.state === backups.BACKUP_STATE_ERROR) { + if (backup.state === backupListing.BACKUP_STATE_ERROR) { backup.discardReason = 'error'; - } else if (backup.state === backups.BACKUP_STATE_CREATING) { + } else if (backup.state === backupListing.BACKUP_STATE_CREATING) { if ((now - backup.creationTime) < 48*60*60*1000) backup.keepReason = 'creating'; else backup.discardReason = 'creating-too-long'; } else if (referencedBackupIds.includes(backup.id)) { // could also be in archives @@ -70,7 +71,7 @@ function applyBackupRetention(allBackups, retention, referencedBackupIds) { } if (retention.keepLatest) { - const latestNormalBackup = allBackups.find(b => b.state === backups.BACKUP_STATE_NORMAL); + const latestNormalBackup = allBackups.find(b => b.state === backupListing.BACKUP_STATE_NORMAL); if (latestNormalBackup && !latestNormalBackup.keepReason) latestNormalBackup.keepReason = 'latest'; } @@ -104,7 +105,7 @@ async function removeBackup(backupConfig, backup, progressCallback) { const [pruneError] = await safe(storage.api(backupConfig.provider).remove(backupConfig, path.dirname(backupFilePath))); if (pruneError) debug(`removeBackup: unable to prune backup directory ${path.dirname(backupFilePath)}: ${pruneError.message}`); - const [delError] = await safe(backups.del(backup.id)); + const [delError] = await safe(backupListing.del(backup.id)); if (delError) debug(`removeBackup: error removing ${backup.id} from database. %o`, delError); else debug(`removeBackup: removed ${backup.remotePath}`); } @@ -121,7 +122,7 @@ async function cleanupAppBackups(backupConfig, retention, referencedBackupIds, p const allAppIds = allApps.map(a => a.id); // high number, try to get all app backups as we had a cloudron with over 100 apps with 4 daily backups for one month! - const appBackups = await backups.getByTypePaged(backups.BACKUP_TYPE_APP, 1, 100000); + const appBackups = await backupListing.getByTypePaged(backupListing.BACKUP_TYPE_APP, 1, 100000); // collate the backups by app id. note that the app could already have been uninstalled const appBackupsById = {}; @@ -158,7 +159,7 @@ async function cleanupMailBackups(backupConfig, retention, referencedBackupIds, const removedMailBackupPaths = []; - const mailBackups = await backups.getByTypePaged(backups.BACKUP_TYPE_MAIL, 1, 100000); + const mailBackups = await backupListing.getByTypePaged(backupListing.BACKUP_TYPE_MAIL, 1, 100000); applyBackupRetention(mailBackups, Object.assign({ keepLatest: true }, retention), referencedBackupIds); @@ -185,7 +186,7 @@ async function cleanupBoxBackups(backupConfig, retention, progressCallback) { // We need to fetch all box backups to be able to compile a list of all referenced app backups. // Otherwise if we miss some app backups, they will get purged! // 100000 here should be seen as infinity - const boxBackups = await backups.getByTypePaged(backups.BACKUP_TYPE_BOX, 1, 100000); + const boxBackups = await backupListing.getByTypePaged(backupListing.BACKUP_TYPE_BOX, 1, 100000); applyBackupRetention(boxBackups, Object.assign({ keepLatest: true }, retention), [] /* references */); @@ -218,10 +219,10 @@ async function cleanupMissingBackups(backupConfig, progressCallback) { let page = 1, result = []; do { - result = await backups.list(page, perPage); + result = await backupListing.list(page, perPage); for (const backup of result) { - if (backup.state !== backups.BACKUP_STATE_NORMAL) continue; // note: errored and incomplete backups are cleaned up by the backup retention logic + if (backup.state !== backupListing.BACKUP_STATE_NORMAL) continue; // note: errored and incomplete backups are cleaned up by the backup retention logic let backupFilePath = backupFormat.api(backupConfig.format).getBackupFilePath(backupConfig, backup.remotePath); if (backupConfig.format === 'rsync') backupFilePath = backupFilePath + '/'; // add trailing slash to indicate directory @@ -231,7 +232,7 @@ async function cleanupMissingBackups(backupConfig, progressCallback) { await progressCallback({ message: `Removing missing backup ${backup.remotePath}`}); - const [delError] = await safe(backups.del(backup.id)); + const [delError] = await safe(backupListing.del(backup.id)); if (delError) debug(`cleanupMissingBackups: error removing ${backup.id} from database. %o`, delError); missingBackupPaths.push(backup.remotePath); diff --git a/src/backuplisting.js b/src/backuplisting.js new file mode 100644 index 000000000..3b3f80fcb --- /dev/null +++ b/src/backuplisting.js @@ -0,0 +1,180 @@ +'use strict'; + +exports = module.exports = { + get, + getByIdentifierAndStatePaged, + getByTypePaged, + add, + update, + setState, + list, + del, + + BACKUP_IDENTIFIER_BOX: 'box', + BACKUP_IDENTIFIER_MAIL: 'mail', + + BACKUP_TYPE_APP: 'app', + BACKUP_TYPE_BOX: 'box', + BACKUP_TYPE_MAIL: 'mail', + + BACKUP_STATE_NORMAL: 'normal', // should rename to created to avoid listing in UI? + BACKUP_STATE_CREATING: 'creating', + BACKUP_STATE_ERROR: 'error', +}; + +const assert = require('assert'), + BoxError = require('./boxerror.js'), + database = require('./database.js'), + hat = require('./hat.js'), + safe = require('safetydance'); + +const BACKUPS_FIELDS = [ 'id', 'remotePath', 'label', 'identifier', 'creationTime', 'packageVersion', 'type', 'dependsOnJson', 'state', 'manifestJson', 'preserveSecs', 'encryptionVersion', 'appConfigJson', 'targetId' ].join(','); + +function postProcess(result) { + assert.strictEqual(typeof result, 'object'); + + result.dependsOn = result.dependsOnJson ? safe.JSON.parse(result.dependsOnJson) : []; + delete result.dependsOnJson; + + result.manifest = result.manifestJson ? safe.JSON.parse(result.manifestJson) : null; + delete result.manifestJson; + + result.appConfig = result.appConfigJson ? safe.JSON.parse(result.appConfigJson) : null; + delete result.appConfigJson; + + return result; +} + +async function add(data) { + assert(data && typeof data === 'object'); + assert.strictEqual(typeof data.remotePath, 'string'); + assert(data.encryptionVersion === null || typeof data.encryptionVersion === 'number'); + assert.strictEqual(typeof data.packageVersion, 'string'); + assert.strictEqual(typeof data.type, 'string'); + assert.strictEqual(typeof data.identifier, 'string'); + assert.strictEqual(typeof data.state, 'string'); + assert(Array.isArray(data.dependsOn)); + assert.strictEqual(typeof data.manifest, 'object'); + assert.strictEqual(typeof data.preserveSecs, 'number'); + assert.strictEqual(typeof data.appConfig, 'object'); + + const creationTime = data.creationTime || new Date(); // allow tests to set the time + const manifestJson = JSON.stringify(data.manifest); + const prefixId = data.type === exports.BACKUP_TYPE_APP ? `${data.type}_${data.identifier}` : data.type; // type and identifier are same for other types + const id = `${prefixId}_v${data.packageVersion}_${hat(32)}`; // id is used by the UI to derive dependent packages. making this a UUID will require a lot of db querying + const appConfigJson = data.appConfig ? JSON.stringify(data.appConfig) : null; + + const targets = await database.query(`SELECT id FROM backupTargets WHERE priority=?`, [ true ]); + const targetId = targets[0].id; + + const [error] = await safe(database.query('INSERT INTO backups (id, remotePath, identifier, encryptionVersion, packageVersion, type, creationTime, state, dependsOnJson, manifestJson, preserveSecs, appConfigJson, targetId) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', + [ id, data.remotePath, data.identifier, data.encryptionVersion, data.packageVersion, data.type, creationTime, data.state, JSON.stringify(data.dependsOn), manifestJson, data.preserveSecs, appConfigJson, targetId ])); + + if (error && error.code === 'ER_DUP_ENTRY') throw new BoxError(BoxError.ALREADY_EXISTS, 'Backup already exists'); + if (error) throw error; + + return id; +} + +async function getByIdentifierAndStatePaged(identifier, state, page, perPage) { + assert.strictEqual(typeof identifier, 'string'); + assert.strictEqual(typeof state, 'string'); + assert(typeof page === 'number' && page > 0); + assert(typeof perPage === 'number' && perPage > 0); + + const results = await database.query(`SELECT ${BACKUPS_FIELDS} FROM backups WHERE identifier = ? AND state = ? ORDER BY creationTime DESC LIMIT ?,?`, [ identifier, state, (page-1)*perPage, perPage ]); + + results.forEach(function (result) { postProcess(result); }); + + return results; +} + +async function get(id) { + assert.strictEqual(typeof id, 'string'); + + const result = await database.query(`SELECT ${BACKUPS_FIELDS} FROM backups WHERE id = ? ORDER BY creationTime DESC`, [ id ]); + if (result.length === 0) return null; + + return postProcess(result[0]); +} + +async function getByTypePaged(type, page, perPage) { + assert.strictEqual(typeof type, 'string'); + assert(typeof page === 'number' && page > 0); + assert(typeof perPage === 'number' && perPage > 0); + + const results = await database.query(`SELECT ${BACKUPS_FIELDS} FROM backups WHERE type = ? ORDER BY creationTime DESC LIMIT ?,?`, [ type, (page-1)*perPage, perPage ]); + + results.forEach(function (result) { postProcess(result); }); + + return results; +} + +function validateLabel(label) { + assert.strictEqual(typeof label, 'string'); + + if (label.length >= 200) return new BoxError(BoxError.BAD_FIELD, 'label too long'); + if (/[^a-zA-Z0-9._() -]/.test(label)) return new BoxError(BoxError.BAD_FIELD, 'label can only contain alphanumerals, space, dot, hyphen, brackets or underscore'); + + return null; +} + +// this is called by REST API +async function update(id, data) { + assert.strictEqual(typeof id, 'string'); + assert.strictEqual(typeof data, 'object'); + + let error; + if ('label' in data) { + error = validateLabel(data.label); + if (error) throw error; + } + + const fields = [], values = []; + for (const p in data) { + if (p === 'label' || p === 'preserveSecs') { + fields.push(p + ' = ?'); + values.push(data[p]); + } + } + values.push(id); + + const backup = await get(id); + if (backup === null) throw new BoxError(BoxError.NOT_FOUND, 'Backup not found'); + + const result = await database.query('UPDATE backups SET ' + fields.join(', ') + ' WHERE id = ?', values); + if (result.affectedRows !== 1) throw new BoxError(BoxError.NOT_FOUND, 'Backup not found'); + + if ('preserveSecs' in data) { + // update the dependancies + for (const depId of backup.dependsOn) { + await database.query('UPDATE backups SET preserveSecs=? WHERE id = ?', [ data.preserveSecs, depId]); + } + } +} + +async function setState(id, state) { + assert.strictEqual(typeof id, 'string'); + assert.strictEqual(typeof state, 'string'); + + const result = await database.query('UPDATE backups SET state = ? WHERE id = ?', [state, id]); + if (result.affectedRows !== 1) throw new BoxError(BoxError.NOT_FOUND, 'Backup not found'); +} + +async function list(page, perPage) { + assert(typeof page === 'number' && page > 0); + assert(typeof perPage === 'number' && perPage > 0); + + const results = await database.query(`SELECT ${BACKUPS_FIELDS} FROM backups ORDER BY creationTime DESC LIMIT ?,?`, [ (page-1)*perPage, perPage ]); + + results.forEach(function (result) { postProcess(result); }); + + return results; +} + +async function del(id) { + assert.strictEqual(typeof id, 'string'); + + const result = await database.query('DELETE FROM backups WHERE id=?', [ id ]); + if (result.affectedRows !== 1) throw new BoxError(BoxError.NOT_FOUND, 'Backup not found'); +} diff --git a/src/backups.js b/src/backups.js index 52cc1deb7..d3be0f843 100644 --- a/src/backups.js +++ b/src/backups.js @@ -1,15 +1,6 @@ 'use strict'; exports = module.exports = { - get, - getByIdentifierAndStatePaged, - getByTypePaged, - add, - update, - setState, - list, - del, - startBackupTask, startCleanupTask, @@ -58,6 +49,7 @@ exports = module.exports = { }; const assert = require('assert'), + backupListing = require('./backuplisting.js'), BoxError = require('./boxerror.js'), constants = require('./constants.js'), cron = require('./cron.js'), @@ -66,7 +58,6 @@ const assert = require('assert'), database = require('./database.js'), debug = require('debug')('box:backups'), eventlog = require('./eventlog.js'), - hat = require('./hat.js'), locks = require('./locks.js'), mounts = require('./mounts.js'), path = require('path'), @@ -78,28 +69,11 @@ const assert = require('assert'), uuid = require('uuid'), _ = require('./underscore.js'); -const BACKUPS_FIELDS = [ 'id', 'remotePath', 'label', 'identifier', 'creationTime', 'packageVersion', 'type', 'dependsOnJson', 'state', 'manifestJson', 'preserveSecs', 'encryptionVersion', 'appConfigJson', 'targetId' ].join(','); - const BACKUP_TARGET_FIELDS = [ 'id', 'label', 'configJson', 'limitsJson', 'retentionJson', 'schedule', 'encryptionJson', 'format', 'priority', 'creationTime', 'ts' ].join(','); function postProcess(result) { assert.strictEqual(typeof result, 'object'); - result.dependsOn = result.dependsOnJson ? safe.JSON.parse(result.dependsOnJson) : []; - delete result.dependsOnJson; - - result.manifest = result.manifestJson ? safe.JSON.parse(result.manifestJson) : null; - delete result.manifestJson; - - result.appConfig = result.appConfigJson ? safe.JSON.parse(result.appConfigJson) : null; - delete result.appConfigJson; - - return result; -} - -function postProcessTarget(result) { - assert.strictEqual(typeof result, 'object'); - result.config = result.configJson ? safe.JSON.parse(result.configJson) : {}; delete result.configJson; @@ -140,80 +114,6 @@ function generateEncryptionKeysSync(password) { }; } -async function add(data) { - assert(data && typeof data === 'object'); - assert.strictEqual(typeof data.remotePath, 'string'); - assert(data.encryptionVersion === null || typeof data.encryptionVersion === 'number'); - assert.strictEqual(typeof data.packageVersion, 'string'); - assert.strictEqual(typeof data.type, 'string'); - assert.strictEqual(typeof data.identifier, 'string'); - assert.strictEqual(typeof data.state, 'string'); - assert(Array.isArray(data.dependsOn)); - assert.strictEqual(typeof data.manifest, 'object'); - assert.strictEqual(typeof data.preserveSecs, 'number'); - assert.strictEqual(typeof data.appConfig, 'object'); - - const creationTime = data.creationTime || new Date(); // allow tests to set the time - const manifestJson = JSON.stringify(data.manifest); - const prefixId = data.type === exports.BACKUP_TYPE_APP ? `${data.type}_${data.identifier}` : data.type; // type and identifier are same for other types - const id = `${prefixId}_v${data.packageVersion}_${hat(32)}`; // id is used by the UI to derive dependent packages. making this a UUID will require a lot of db querying - const appConfigJson = data.appConfig ? JSON.stringify(data.appConfig) : null; - - const targets = await database.query(`SELECT id FROM backupTargets WHERE priority=?`, [ true ]); - const targetId = targets[0].id; - - const [error] = await safe(database.query('INSERT INTO backups (id, remotePath, identifier, encryptionVersion, packageVersion, type, creationTime, state, dependsOnJson, manifestJson, preserveSecs, appConfigJson, targetId) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', - [ id, data.remotePath, data.identifier, data.encryptionVersion, data.packageVersion, data.type, creationTime, data.state, JSON.stringify(data.dependsOn), manifestJson, data.preserveSecs, appConfigJson, targetId ])); - - if (error && error.code === 'ER_DUP_ENTRY') throw new BoxError(BoxError.ALREADY_EXISTS, 'Backup already exists'); - if (error) throw error; - - return id; -} - -async function getByIdentifierAndStatePaged(identifier, state, page, perPage) { - assert.strictEqual(typeof identifier, 'string'); - assert.strictEqual(typeof state, 'string'); - assert(typeof page === 'number' && page > 0); - assert(typeof perPage === 'number' && perPage > 0); - - const results = await database.query(`SELECT ${BACKUPS_FIELDS} FROM backups WHERE identifier = ? AND state = ? ORDER BY creationTime DESC LIMIT ?,?`, [ identifier, state, (page-1)*perPage, perPage ]); - - results.forEach(function (result) { postProcess(result); }); - - return results; -} - -async function get(id) { - assert.strictEqual(typeof id, 'string'); - - const result = await database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups WHERE id = ? ORDER BY creationTime DESC', [ id ]); - if (result.length === 0) return null; - - return postProcess(result[0]); -} - -async function getByTypePaged(type, page, perPage) { - assert.strictEqual(typeof type, 'string'); - assert(typeof page === 'number' && page > 0); - assert(typeof perPage === 'number' && perPage > 0); - - const results = await database.query(`SELECT ${BACKUPS_FIELDS} FROM backups WHERE type = ? ORDER BY creationTime DESC LIMIT ?,?`, [ type, (page-1)*perPage, perPage ]); - - results.forEach(function (result) { postProcess(result); }); - - return results; -} - -function validateLabel(label) { - assert.strictEqual(typeof label, 'string'); - - if (label.length >= 200) return new BoxError(BoxError.BAD_FIELD, 'label too long'); - if (/[^a-zA-Z0-9._() -]/.test(label)) return new BoxError(BoxError.BAD_FIELD, 'label can only contain alphanumerals, space, dot, hyphen, brackets or underscore'); - - return null; -} - async function validatePolicy(policy) { assert.strictEqual(typeof policy, 'object'); @@ -230,48 +130,6 @@ async function validatePolicy(policy) { if ('keepYearly' in retention && typeof retention.keepYearly !== 'number') return new BoxError(BoxError.BAD_FIELD, 'retention.keepYearly must be a number'); } -// this is called by REST API -async function update(id, data) { - assert.strictEqual(typeof id, 'string'); - assert.strictEqual(typeof data, 'object'); - - let error; - if ('label' in data) { - error = validateLabel(data.label); - if (error) throw error; - } - - const fields = [], values = []; - for (const p in data) { - if (p === 'label' || p === 'preserveSecs') { - fields.push(p + ' = ?'); - values.push(data[p]); - } - } - values.push(id); - - const backup = await get(id); - if (backup === null) throw new BoxError(BoxError.NOT_FOUND, 'Backup not found'); - - const result = await database.query('UPDATE backups SET ' + fields.join(', ') + ' WHERE id = ?', values); - if (result.affectedRows !== 1) throw new BoxError(BoxError.NOT_FOUND, 'Backup not found'); - - if ('preserveSecs' in data) { - // update the dependancies - for (const depId of backup.dependsOn) { - await database.query('UPDATE backups SET preserveSecs=? WHERE id = ?', [ data.preserveSecs, depId]); - } - } -} - -async function setState(id, state) { - assert.strictEqual(typeof id, 'string'); - assert.strictEqual(typeof state, 'string'); - - const result = await database.query('UPDATE backups SET state = ? WHERE id = ?', [state, id]); - if (result.affectedRows !== 1) throw new BoxError(BoxError.NOT_FOUND, 'Backup not found'); -} - async function startBackupTask(auditSource) { const [error] = await safe(locks.acquire(locks.TYPE_FULL_BACKUP_TASK)); if (error) throw new BoxError(BoxError.BAD_STATE, `Another backup task is in progress: ${error.message}`); @@ -287,7 +145,7 @@ async function startBackupTask(auditSource) { // background tasks.startTask(taskId, { timeout: 24 * 60 * 60 * 1000 /* 24 hours */, nice: 15, memoryLimit, oomScoreAdjust: -999 }) .then(async (backupId) => { - const backup = await get(backupId); + const backup = await backupListing.get(backupId); await eventlog.add(eventlog.ACTION_BACKUP_FINISH, auditSource, { taskId, backupId, remotePath: backup.remotePath }); }) .catch(async (error) => { @@ -302,24 +160,6 @@ async function startBackupTask(auditSource) { return taskId; } -async function list(page, perPage) { - assert(typeof page === 'number' && page > 0); - assert(typeof perPage === 'number' && perPage > 0); - - const results = await database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups ORDER BY creationTime DESC LIMIT ?,?', [ (page-1)*perPage, perPage ]); - - results.forEach(function (result) { postProcess(result); }); - - return results; -} - -async function del(id) { - assert.strictEqual(typeof id, 'string'); - - const result = await database.query('DELETE FROM backups WHERE id=?', [ id ]); - if (result.affectedRows !== 1) throw new BoxError(BoxError.NOT_FOUND, 'Backup not found'); -} - // this function is used in migrations - 20200512172301-settings-backup-encryption.js function cleanupCacheFilesSync() { const files = safe.fs.readdirSync(path.join(paths.BACKUP_INFO_DIR)); @@ -433,7 +273,7 @@ async function ensureMounted() { async function getPolicy() { const results = await database.query(`SELECT ${BACKUP_TARGET_FIELDS} FROM backupTargets WHERE priority=?`, [ true ]); - const result = postProcessTarget(results[0]); + const result = postProcess(results[0]); return { retention: result.retention, schedule: result.schedule }; } @@ -501,12 +341,12 @@ async function addDefaultTarget() { async function getTarget(id) { const results = await database.query(`SELECT ${BACKUP_TARGET_FIELDS} FROM backupTargets WHERE id=?`, [ id ]); if (results.length === 0) return null; - return postProcessTarget(results[0]); + return postProcess(results[0]); } async function getConfig() { const results = await database.query(`SELECT ${BACKUP_TARGET_FIELDS} FROM backupTargets WHERE priority=?`, [ true ]); - const result = postProcessTarget(results[0]); + const result = postProcess(results[0]); const config = result.config; config.format = result.format; diff --git a/src/backuptask.js b/src/backuptask.js index beba3a284..5d4eed79c 100644 --- a/src/backuptask.js +++ b/src/backuptask.js @@ -18,6 +18,7 @@ exports = module.exports = { const apps = require('./apps.js'), assert = require('assert'), backupFormat = require('./backupformat.js'), + backupListing = require('./backuplisting.js'), backups = require('./backups.js'), BoxError = require('./boxerror.js'), constants = require('./constants.js'), @@ -233,19 +234,19 @@ async function rotateBoxBackup(backupConfig, tag, options, dependsOn, progressCa remotePath, encryptionVersion: backupConfig.encryption ? 2 : null, packageVersion: constants.VERSION, - type: backups.BACKUP_TYPE_BOX, - state: backups.BACKUP_STATE_CREATING, - identifier: backups.BACKUP_IDENTIFIER_BOX, + type: backupListing.BACKUP_TYPE_BOX, + state: backupListing.BACKUP_STATE_CREATING, + identifier: backupListing.BACKUP_IDENTIFIER_BOX, dependsOn, manifest: null, preserveSecs: options.preserveSecs || 0, appConfig: null }; - const id = await backups.add(data); + const id = await backupListing.add(data); const [error] = await safe(copy(backupConfig, 'snapshot/box', remotePath, progressCallback)); - const state = error ? backups.BACKUP_STATE_ERROR : backups.BACKUP_STATE_NORMAL; - await backups.setState(id, state); + const state = error ? backupListing.BACKUP_STATE_ERROR : backupListing.BACKUP_STATE_NORMAL; + await backupListing.setState(id, state); if (error) throw error; return id; @@ -281,8 +282,8 @@ async function rotateAppBackup(backupConfig, app, tag, options, progressCallback remotePath, encryptionVersion: backupConfig.encryption ? 2 : null, packageVersion: manifest.version, - type: backups.BACKUP_TYPE_APP, - state: backups.BACKUP_STATE_CREATING, + type: backupListing.BACKUP_TYPE_APP, + state: backupListing.BACKUP_STATE_CREATING, identifier: app.id, dependsOn: [], manifest, @@ -290,10 +291,10 @@ async function rotateAppBackup(backupConfig, app, tag, options, progressCallback appConfig: app }; - const id = await backups.add(data); + const id = await backupListing.add(data); const [error] = await safe(copy(backupConfig, `snapshot/app_${app.id}`, remotePath, progressCallback)); - const state = error ? backups.BACKUP_STATE_ERROR : backups.BACKUP_STATE_NORMAL; - await backups.setState(id, state); + const state = error ? backupListing.BACKUP_STATE_ERROR : backupListing.BACKUP_STATE_NORMAL; + await backupListing.setState(id, state); if (error) throw error; return id; @@ -368,7 +369,7 @@ async function backupAppWithTag(app, tag, options, progressCallback) { assert.strictEqual(typeof progressCallback, 'function'); if (!apps.canBackupApp(app)) { // if we cannot backup, reuse it's most recent backup - const results = await backups.getByIdentifierAndStatePaged(app.id, backups.BACKUP_STATE_NORMAL, 1, 1); + const results = await backupListing.getByIdentifierAndStatePaged(app.id, backupListing.BACKUP_STATE_NORMAL, 1, 1); if (results.length === 0) return null; // no backup to re-use return results[0].id; @@ -419,19 +420,19 @@ async function rotateMailBackup(backupConfig, tag, options, progressCallback) { remotePath, encryptionVersion: backupConfig.encryption ? 2 : null, packageVersion: constants.VERSION, - type: backups.BACKUP_TYPE_MAIL, - state: backups.BACKUP_STATE_CREATING, - identifier: backups.BACKUP_IDENTIFIER_MAIL, + type: backupListing.BACKUP_TYPE_MAIL, + state: backupListing.BACKUP_STATE_CREATING, + identifier: backupListing.BACKUP_IDENTIFIER_MAIL, dependsOn: [], manifest: null, preserveSecs: options.preserveSecs || 0, appConfig: null }; - const id = await backups.add(data); + const id = await backupListing.add(data); const [error] = await safe(copy(backupConfig, 'snapshot/mail', remotePath, progressCallback)); - const state = error ? backups.BACKUP_STATE_ERROR : backups.BACKUP_STATE_NORMAL; - await backups.setState(id, state); + const state = error ? backupListing.BACKUP_STATE_ERROR : backupListing.BACKUP_STATE_NORMAL; + await backupListing.setState(id, state); if (error) throw error; return id; diff --git a/src/provision.js b/src/provision.js index 62df2e202..1ab26b292 100644 --- a/src/provision.js +++ b/src/provision.js @@ -10,6 +10,7 @@ exports = module.exports = { const appstore = require('./appstore.js'), assert = require('assert'), backups = require('./backups.js'), + backupListing = require('./backuplisting.js'), backuptask = require('./backuptask.js'), BoxError = require('./boxerror.js'), dashboard = require('./dashboard.js'), @@ -184,7 +185,7 @@ async function restoreTask(backupConfig, remotePath, ipv4Config, ipv6Config, opt await backuptask.restore(backupConfig, remotePath, (progress) => setProgress('restore', progress.message)); setProgress('restore', 'Downloading mail backup'); - const mailBackups = await backups.getByIdentifierAndStatePaged(backups.BACKUP_IDENTIFIER_MAIL, backups.BACKUP_STATE_NORMAL, 1, 1); + const mailBackups = await backupListing.getByIdentifierAndStatePaged(backupListing.BACKUP_IDENTIFIER_MAIL, backupListing.BACKUP_STATE_NORMAL, 1, 1); if (mailBackups.length === 0) throw new BoxError(BoxError.NOT_FOUND, 'mail backup not found'); const mailRestoreConfig = { backupConfig, remotePath: mailBackups[0].remotePath, backupFormat: mailBackups[0].format }; await backuptask.downloadMail(mailRestoreConfig, (progress) => setProgress('restore', progress.message)); diff --git a/src/routes/backuplisting.js b/src/routes/backuplisting.js new file mode 100644 index 000000000..a6be02bf5 --- /dev/null +++ b/src/routes/backuplisting.js @@ -0,0 +1,40 @@ +'use strict'; + +exports = module.exports = { + list, + update, +}; + +const assert = require('assert'), + backupListing = require('../backuplisting.js'), + BoxError = require('../boxerror.js'), + HttpError = require('@cloudron/connect-lastmile').HttpError, + HttpSuccess = require('@cloudron/connect-lastmile').HttpSuccess, + safe = require('safetydance'); + +async function list(req, res, next) { + const page = typeof req.query.page === 'string' ? parseInt(req.query.page) : 1; + if (!page || page < 0) return next(new HttpError(400, 'page query param has to be a postive number')); + + const perPage = typeof req.query.per_page === 'string'? parseInt(req.query.per_page) : 25; + if (!perPage || perPage < 0) return next(new HttpError(400, 'per_page query param has to be a postive number')); + + const [error, result] = await safe(backupListing.getByIdentifierAndStatePaged(backupListing.BACKUP_IDENTIFIER_BOX, backupListing.BACKUP_STATE_NORMAL, page, perPage)); + if (error) return next(BoxError.toHttpError(error)); + + next(new HttpSuccess(200, { backups: result })); +} + +async function update(req, res, next) { + assert.strictEqual(typeof req.params.backupId, 'string'); + assert.strictEqual(typeof req.body, 'object'); + + const { label, preserveSecs } = req.body; + if (typeof label !== 'string') return next(new HttpError(400, 'label must be a string')); + if (typeof preserveSecs !== 'number') return next(new HttpError(400, 'preserveSecs must be a number')); + + const [error] = await safe(backupListing.update(req.params.backupId, { label, preserveSecs })); + if (error) return next(BoxError.toHttpError(error)); + + next(new HttpSuccess(200, {})); +} diff --git a/src/routes/backups.js b/src/routes/backups.js index 85301068f..daf4f2012 100644 --- a/src/routes/backups.js +++ b/src/routes/backups.js @@ -1,8 +1,6 @@ 'use strict'; exports = module.exports = { - list, - update, create, cleanup, remount, @@ -24,33 +22,6 @@ const assert = require('assert'), HttpSuccess = require('@cloudron/connect-lastmile').HttpSuccess, safe = require('safetydance'); -async function list(req, res, next) { - const page = typeof req.query.page === 'string' ? parseInt(req.query.page) : 1; - if (!page || page < 0) return next(new HttpError(400, 'page query param has to be a postive number')); - - const perPage = typeof req.query.per_page === 'string'? parseInt(req.query.per_page) : 25; - if (!perPage || perPage < 0) return next(new HttpError(400, 'per_page query param has to be a postive number')); - - const [error, result] = await safe(backups.getByIdentifierAndStatePaged(backups.BACKUP_IDENTIFIER_BOX, backups.BACKUP_STATE_NORMAL, page, perPage)); - if (error) return next(BoxError.toHttpError(error)); - - next(new HttpSuccess(200, { backups: result })); -} - -async function update(req, res, next) { - assert.strictEqual(typeof req.params.backupId, 'string'); - assert.strictEqual(typeof req.body, 'object'); - - const { label, preserveSecs } = req.body; - if (typeof label !== 'string') return next(new HttpError(400, 'label must be a string')); - if (typeof preserveSecs !== 'number') return next(new HttpError(400, 'preserveSecs must be a number')); - - const [error] = await safe(backups.update(req.params.backupId, { label, preserveSecs })); - if (error) return next(BoxError.toHttpError(error)); - - next(new HttpSuccess(200, {})); -} - async function create(req, res, next) { const [error, taskId] = await safe(backups.startBackupTask(AuditSource.fromRequest(req))); if (error) return next(BoxError.toHttpError(error)); diff --git a/src/routes/index.js b/src/routes/index.js index 349bc4935..a190a3b34 100644 --- a/src/routes/index.js +++ b/src/routes/index.js @@ -8,6 +8,7 @@ exports = module.exports = { appstore: require('./appstore.js'), archives: require('./archives.js'), auth: require('./auth.js'), + backupListing: require('./backuplisting.js'), backups: require('./backups.js'), branding: require('./branding.js'), cloudron: require('./cloudron.js'), diff --git a/src/routes/test/archives-test.js b/src/routes/test/archives-test.js index e4e9d0895..ede24916d 100644 --- a/src/routes/test/archives-test.js +++ b/src/routes/test/archives-test.js @@ -6,7 +6,7 @@ 'use strict'; const archives = require('../../archives.js'), - backups = require('../../backups.js'), + backupListing = require('../../backuplisting.js'), common = require('./common.js'), expect = require('expect.js'), superagent = require('@cloudron/superagent'); @@ -19,8 +19,8 @@ describe('Archives API', function () { remotePath: 'app_appid_123', encryptionVersion: null, packageVersion: '1.0.0', - type: backups.BACKUP_TYPE_APP, - state: backups.BACKUP_STATE_CREATING, + type: backupListing.BACKUP_TYPE_APP, + state: backupListing.BACKUP_STATE_CREATING, identifier: 'appid', dependsOn: [ ], manifest: { foo: 'bar' }, @@ -33,7 +33,7 @@ describe('Archives API', function () { before(async function () { await setup(); - appBackup.id = await backups.add(appBackup); + appBackup.id = await backupListing.add(appBackup); archiveId = await archives.add(appBackup.id, {}, auditSource); }); after(cleanup); diff --git a/src/server.js b/src/server.js index 64efadf86..4f406e669 100644 --- a/src/server.js +++ b/src/server.js @@ -152,7 +152,9 @@ async function initializeExpressSync() { router.post('/api/v1/notifications/:notificationId', json, token, authorizeAdmin, routes.notifications.load, routes.notifications.update); // backup routes - router.get ('/api/v1/backups', token, authorizeAdmin, routes.backups.list); + router.get ('/api/v1/backups', token, authorizeAdmin, routes.backupListing.list); + router.post('/api/v1/backups/:backupId', json, token, authorizeAdmin, routes.backupListing.update); + router.get ('/api/v1/backups/mount_status', token, authorizeAdmin, routes.backups.getMountStatus); router.post('/api/v1/backups/create', token, authorizeAdmin, routes.backups.create); router.post('/api/v1/backups/cleanup', json, token, authorizeAdmin, routes.backups.cleanup); @@ -162,7 +164,6 @@ async function initializeExpressSync() { router.post('/api/v1/backups/config/limits', json, token, authorizeOwner, routes.backups.setLimits); router.get ('/api/v1/backups/policy', token, authorizeAdmin, routes.backups.getPolicy); router.post('/api/v1/backups/policy', json, token, authorizeOwner, routes.backups.setPolicy); - router.post('/api/v1/backups/:backupId', json, token, authorizeAdmin, routes.backups.update); // app archive routes router.get ('/api/v1/archives', token, authorizeAdmin, routes.archives.list); diff --git a/src/test/archives-test.js b/src/test/archives-test.js index 8551ca228..2ff04179b 100644 --- a/src/test/archives-test.js +++ b/src/test/archives-test.js @@ -6,7 +6,7 @@ 'use strict'; const archives = require('../archives.js'), - backups = require('../backups.js'), + backupListing = require('../backuplisting.js'), BoxError = require('../boxerror.js'), common = require('./common.js'), expect = require('expect.js'), @@ -20,8 +20,8 @@ describe('Archives', function () { remotePath: 'backup-box', encryptionVersion: 2, packageVersion: '1.0.0', - type: backups.BACKUP_TYPE_APP, - state: backups.BACKUP_STATE_NORMAL, + type: backupListing.BACKUP_TYPE_APP, + state: backupListing.BACKUP_STATE_NORMAL, identifier: 'box', dependsOn: [ 'dep1' ], manifest: null, @@ -32,7 +32,7 @@ describe('Archives', function () { before(async function () { await setup(); - appBackup.id = await backups.add(appBackup); + appBackup.id = await backupListing.add(appBackup); }); after(cleanup); diff --git a/src/test/backupcleaner-test.js b/src/test/backupcleaner-test.js index e5426d9e6..49d3a7594 100644 --- a/src/test/backupcleaner-test.js +++ b/src/test/backupcleaner-test.js @@ -8,6 +8,7 @@ const archives = require('../archives.js'), backupCleaner = require('../backupcleaner.js'), + backupListing = require('../backuplisting.js'), backups = require('../backups.js'), common = require('./common.js'), expect = require('expect.js'), @@ -27,8 +28,8 @@ describe('backup cleaner', function () { remotePath: 'somepath', encryptionVersion: 2, packageVersion: '1.0.0', - type: backups.BACKUP_TYPE_BOX, - state: backups.BACKUP_STATE_NORMAL, + type: backupListing.BACKUP_TYPE_BOX, + state: backupListing.BACKUP_STATE_NORMAL, identifier: 'box', dependsOn: [ 'dep1' ], manifest: null, @@ -38,13 +39,13 @@ describe('backup cleaner', function () { describe('retention', function () { it('keeps latest', function () { - const backup = Object.assign({}, backupTemplate, { creationTime: moment().subtract(5, 's').toDate(), state: backups.BACKUP_STATE_NORMAL }); + const backup = Object.assign({}, backupTemplate, { creationTime: moment().subtract(5, 's').toDate(), state: backupListing.BACKUP_STATE_NORMAL }); backupCleaner._applyBackupRetention([backup], { keepWithinSecs: 1, keepLatest: true }, []); expect(backup.keepReason).to.be('latest'); }); it('does not keep latest', function () { - const backup = { creationTime: moment().subtract(5, 's').toDate(), state: backups.BACKUP_STATE_NORMAL }; + const backup = { creationTime: moment().subtract(5, 's').toDate(), state: backupListing.BACKUP_STATE_NORMAL }; backupCleaner._applyBackupRetention([backup], { keepWithinSecs: 1, keepLatest: false }, []); expect(backup.keepReason).to.be(undefined); }); @@ -63,11 +64,11 @@ describe('backup cleaner', function () { it('1 daily', function () { const b = [ - { id: '0', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().toDate() }, - { id: '1', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(1, 'h').toDate() }, - { id: '2', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(3, 'h').toDate() }, - { id: '3', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(20, 'h').toDate() }, - { id: '4', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(5, 'd').toDate() } + { id: '0', state: backupListing.BACKUP_STATE_NORMAL, creationTime: moment().toDate() }, + { id: '1', state: backupListing.BACKUP_STATE_NORMAL, creationTime: moment().subtract(1, 'h').toDate() }, + { id: '2', state: backupListing.BACKUP_STATE_NORMAL, creationTime: moment().subtract(3, 'h').toDate() }, + { id: '3', state: backupListing.BACKUP_STATE_NORMAL, creationTime: moment().subtract(20, 'h').toDate() }, + { id: '4', state: backupListing.BACKUP_STATE_NORMAL, creationTime: moment().subtract(5, 'd').toDate() } ]; backupCleaner._applyBackupRetention(b, { keepDaily: 1, keepLatest: true }, []); expect(b[0].keepReason).to.be('keepDaily'); @@ -80,13 +81,13 @@ describe('backup cleaner', function () { // if you are debugging this test, it's because of some timezone issue with all the hour substraction! it('2 daily, 1 weekly', function () { const b = [ - { id: '0', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().toDate() }, - { id: '1', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(1, 'h').toDate() }, - { id: '2', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(3, 'h').toDate() }, - { id: '3', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(26, 'h').toDate() }, - { id: '4', state: backups.BACKUP_STATE_ERROR, creationTime: moment().subtract(32, 'h').toDate() }, - { id: '5', state: backups.BACKUP_STATE_CREATING, creationTime: moment().subtract(50, 'h').toDate() }, - { id: '6', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(5, 'd').toDate() } + { id: '0', state: backupListing.BACKUP_STATE_NORMAL, creationTime: moment().toDate() }, + { id: '1', state: backupListing.BACKUP_STATE_NORMAL, creationTime: moment().subtract(1, 'h').toDate() }, + { id: '2', state: backupListing.BACKUP_STATE_NORMAL, creationTime: moment().subtract(3, 'h').toDate() }, + { id: '3', state: backupListing.BACKUP_STATE_NORMAL, creationTime: moment().subtract(26, 'h').toDate() }, + { id: '4', state: backupListing.BACKUP_STATE_ERROR, creationTime: moment().subtract(32, 'h').toDate() }, + { id: '5', state: backupListing.BACKUP_STATE_CREATING, creationTime: moment().subtract(50, 'h').toDate() }, + { id: '6', state: backupListing.BACKUP_STATE_NORMAL, creationTime: moment().subtract(5, 'd').toDate() } ]; backupCleaner._applyBackupRetention(b, { keepDaily: 2, keepWeekly: 1, keepLatest: false }, []); expect(b[0].keepReason).to.be('keepDaily'); // today @@ -100,16 +101,16 @@ describe('backup cleaner', function () { it('2 daily, 3 monthly, 1 yearly', function () { const b = [ - { id: '0', state: backups.BACKUP_STATE_CREATING, creationTime: moment().toDate() }, - { id: '1', state: backups.BACKUP_STATE_ERROR, creationTime: moment().toDate() }, - { id: '2', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().toDate() }, - { id: '3', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(1, 'h').toDate() }, - { id: '4', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(3, 'h').toDate() }, - { id: '5', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(26, 'h').toDate() }, - { id: '6', state: backups.BACKUP_STATE_CREATING, creationTime: moment().subtract(49, 'h').toDate() }, - { id: '7', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(51, 'd').toDate() }, - { id: '8', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(84, 'd').toDate() }, - { id: '9', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(97, 'd').toDate() }, + { id: '0', state: backupListing.BACKUP_STATE_CREATING, creationTime: moment().toDate() }, + { id: '1', state: backupListing.BACKUP_STATE_ERROR, creationTime: moment().toDate() }, + { id: '2', state: backupListing.BACKUP_STATE_NORMAL, creationTime: moment().toDate() }, + { id: '3', state: backupListing.BACKUP_STATE_NORMAL, creationTime: moment().subtract(1, 'h').toDate() }, + { id: '4', state: backupListing.BACKUP_STATE_NORMAL, creationTime: moment().subtract(3, 'h').toDate() }, + { id: '5', state: backupListing.BACKUP_STATE_NORMAL, creationTime: moment().subtract(26, 'h').toDate() }, + { id: '6', state: backupListing.BACKUP_STATE_CREATING, creationTime: moment().subtract(49, 'h').toDate() }, + { id: '7', state: backupListing.BACKUP_STATE_NORMAL, creationTime: moment().subtract(51, 'd').toDate() }, + { id: '8', state: backupListing.BACKUP_STATE_NORMAL, creationTime: moment().subtract(84, 'd').toDate() }, + { id: '9', state: backupListing.BACKUP_STATE_NORMAL, creationTime: moment().subtract(97, 'd').toDate() }, ]; backupCleaner._applyBackupRetention(b, { keepDaily: 2, keepMonthly: 3, keepYearly: 1, keepLatest: true }, []); expect(b[0].keepReason).to.be('creating'); @@ -132,8 +133,8 @@ describe('backup cleaner', function () { identifier: 'box', encryptionVersion: null, packageVersion: '1.0.0', - type: backups.BACKUP_TYPE_BOX, - state: backups.BACKUP_STATE_NORMAL, + type: backupListing.BACKUP_TYPE_BOX, + state: backupListing.BACKUP_STATE_NORMAL, dependsOn: [ 'backup-app-00', 'backup-app-01' ], manifest: null, preserveSecs: 0, @@ -146,8 +147,8 @@ describe('backup cleaner', function () { identifier: app.id, encryptionVersion: null, packageVersion: '1.0.0', - type: backups.BACKUP_TYPE_APP, - state: backups.BACKUP_STATE_NORMAL, + type: backupListing.BACKUP_TYPE_APP, + state: backupListing.BACKUP_STATE_NORMAL, dependsOn: [], manifest: null, preserveSecs: 0, @@ -160,8 +161,8 @@ describe('backup cleaner', function () { identifier: 'app1', encryptionVersion: null, packageVersion: '1.0.0', - type: backups.BACKUP_TYPE_APP, - state: backups.BACKUP_STATE_NORMAL, + type: backupListing.BACKUP_TYPE_APP, + state: backupListing.BACKUP_STATE_NORMAL, dependsOn: [], manifest: null, preserveSecs: 0, @@ -173,8 +174,8 @@ describe('backup cleaner', function () { remotePath: 'backup-box-1', encryptionVersion: null, packageVersion: '1.0.0', - type: backups.BACKUP_TYPE_BOX, - state: backups.BACKUP_STATE_NORMAL, + type: backupListing.BACKUP_TYPE_BOX, + state: backupListing.BACKUP_STATE_NORMAL, identifier: 'box', dependsOn: [ 'backup-app-10', 'backup-app-11' ], manifest: null, @@ -187,8 +188,8 @@ describe('backup cleaner', function () { remotePath: 'backup-app-10', encryptionVersion: null, packageVersion: '1.0.0', - type: backups.BACKUP_TYPE_APP, - state: backups.BACKUP_STATE_NORMAL, + type: backupListing.BACKUP_TYPE_APP, + state: backupListing.BACKUP_STATE_NORMAL, identifier: app.id, dependsOn: [], manifest: null, @@ -201,8 +202,8 @@ describe('backup cleaner', function () { remotePath: 'backup-app-11', encryptionVersion: null, packageVersion: '1.0.0', - type: backups.BACKUP_TYPE_APP, - state: backups.BACKUP_STATE_NORMAL, + type: backupListing.BACKUP_TYPE_APP, + state: backupListing.BACKUP_STATE_NORMAL, identifier: 'app1', dependsOn: [], manifest: null, @@ -215,8 +216,8 @@ describe('backup cleaner', function () { remotePath: 'backup-app-2', encryptionVersion: null, packageVersion: '2.0.0', - type: backups.BACKUP_TYPE_APP, - state: backups.BACKUP_STATE_NORMAL, + type: backupListing.BACKUP_TYPE_APP, + state: backupListing.BACKUP_STATE_NORMAL, identifier: 'app2', dependsOn: [], manifest: null, @@ -256,56 +257,56 @@ describe('backup cleaner', function () { }); it('add the backups', async function () { - BACKUP_0_APP_0.id = await backups.add(BACKUP_0_APP_0); - BACKUP_0_APP_1.id = await backups.add(BACKUP_0_APP_1); + BACKUP_0_APP_0.id = await backupListing.add(BACKUP_0_APP_0); + BACKUP_0_APP_1.id = await backupListing.add(BACKUP_0_APP_1); BACKUP_0_BOX.dependsOn = [ BACKUP_0_APP_0.id, BACKUP_0_APP_1.id ]; - BACKUP_0_BOX.id = await backups.add(BACKUP_0_BOX); + BACKUP_0_BOX.id = await backupListing.add(BACKUP_0_BOX); await timers.setTimeout(2000); // space out backups - BACKUP_1_APP_0.id = await backups.add(BACKUP_1_APP_0); - BACKUP_1_APP_1.id = await backups.add(BACKUP_1_APP_1); + BACKUP_1_APP_0.id = await backupListing.add(BACKUP_1_APP_0); + BACKUP_1_APP_1.id = await backupListing.add(BACKUP_1_APP_1); BACKUP_1_BOX.dependsOn = [ BACKUP_1_APP_0.id, BACKUP_1_APP_1.id ]; - BACKUP_1_BOX.id = await backups.add(BACKUP_1_BOX); + BACKUP_1_BOX.id = await backupListing.add(BACKUP_1_BOX); - BACKUP_2_APP_2.id = await backups.add(BACKUP_2_APP_2); + BACKUP_2_APP_2.id = await backupListing.add(BACKUP_2_APP_2); await archives.add(BACKUP_2_APP_2.id, {}, common.auditSource); }); it('succeeds with box backups, keeps latest', async function () { await cleanupBackups(); - const results = await backups.getByTypePaged(backups.BACKUP_TYPE_BOX, 1, 1000); + const results = await backupListing.getByTypePaged(backupListing.BACKUP_TYPE_BOX, 1, 1000); expect(results.length).to.equal(1); expect(results[0].id).to.equal(BACKUP_1_BOX.id); // check that app backups are gone as well. only backup_1 will remain - const result = await backups.get(BACKUP_0_APP_0.id); + const result = await backupListing.get(BACKUP_0_APP_0.id); expect(result).to.be(null); }); it('does not remove expired backups if only one left', async function () { await cleanupBackups(); - const results = await backups.getByTypePaged(backups.BACKUP_TYPE_BOX, 1, 1000); + const results = await backupListing.getByTypePaged(backupListing.BACKUP_TYPE_BOX, 1, 1000); expect(results[0].id).to.equal(BACKUP_1_BOX.id); // check that app backups are also still there. backup_1 is still there - const result = await backups.get(BACKUP_1_APP_0.id); + const result = await backupListing.get(BACKUP_1_APP_0.id); expect(result.id).to.equal(BACKUP_1_APP_0.id); }); it('succeeds for app backups not referenced by a box backup', async function () { // add two dangling app backups not referenced by box backup. app1 is uninstalled. app0 is there for (const backup of [BACKUP_0_APP_0, BACKUP_0_APP_1]) { - backup.id = await backups.add(backup); + backup.id = await backupListing.add(backup); } await timers.setTimeout(2000); // wait for expiration await cleanupBackups(); - let result = await backups.getByTypePaged(backups.BACKUP_TYPE_APP, 1, 1000); + let result = await backupListing.getByTypePaged(backupListing.BACKUP_TYPE_APP, 1, 1000); expect(result.length).to.equal(4); result = result.sort((r1, r2) => r1.remotePath.localeCompare(r2.remotePath)); expect(result[0].id).to.be(BACKUP_0_APP_0.id); // because app is installed, latest backup is preserved diff --git a/src/test/backups-test.js b/src/test/backups-test.js index e2bc57a51..b0e7d73dc 100644 --- a/src/test/backups-test.js +++ b/src/test/backups-test.js @@ -6,7 +6,8 @@ 'use strict'; -const backups = require('../backups.js'), +const backupListing = require('../backuplisting.js'), + backups = require('../backups.js'), BoxError = require('../boxerror.js'), common = require('./common.js'), expect = require('expect.js'), @@ -20,8 +21,8 @@ describe('backups', function () { remotePath: 'backup-box', encryptionVersion: 2, packageVersion: '1.0.0', - type: backups.BACKUP_TYPE_BOX, - state: backups.BACKUP_STATE_NORMAL, + type: backupListing.BACKUP_TYPE_BOX, + state: backupListing.BACKUP_STATE_NORMAL, identifier: 'box', dependsOn: [ 'dep1' ], manifest: null, @@ -36,8 +37,8 @@ describe('backups', function () { remotePath: 'app_appid_123', encryptionVersion: null, packageVersion: '1.0.0', - type: backups.BACKUP_TYPE_APP, - state: backups.BACKUP_STATE_CREATING, + type: backupListing.BACKUP_TYPE_APP, + state: backupListing.BACKUP_STATE_CREATING, identifier: 'appid', dependsOn: [ ], manifest: { foo: 'bar' }, @@ -57,65 +58,65 @@ describe('backups', function () { describe('crud', function () { it('add succeeds', async function () { - boxBackup.id = await backups.add(boxBackup); + boxBackup.id = await backupListing.add(boxBackup); }); it('fails with duplicate path', async function () { - const [error] = await safe(backups.add(boxBackup)); + const [error] = await safe(backupListing.add(boxBackup)); expect(error.reason).to.be(BoxError.ALREADY_EXISTS); }); it('get succeeds', async function () { - const result = await backups.get(boxBackup.id); + const result = await backupListing.get(boxBackup.id); delete result.creationTime; expect(result).to.eql(boxBackup); }); it('get of unknown id fails', async function () { - const result = await backups.get('somerandom'); + const result = await backupListing.get('somerandom'); expect(result).to.be(null); }); it('getByTypePaged succeeds', async function () { - const results = await backups.getByTypePaged(backups.BACKUP_TYPE_BOX, 1, 5); + const results = await backupListing.getByTypePaged(backupListing.BACKUP_TYPE_BOX, 1, 5); expect(results.length).to.be(1); delete results[0].creationTime; expect(results[0]).to.eql(boxBackup); }); it('update succeeds', async function () { - await backups.update(boxBackup.id, { label: 'DuMonde', preserveSecs: 30 }); - const result = await backups.get(boxBackup.id); + await backupListing.update(boxBackup.id, { label: 'DuMonde', preserveSecs: 30 }); + const result = await backupListing.get(boxBackup.id); expect(result.label).to.eql('DuMonde'); expect(result.preserveSecs).to.eql(30); }); it('delete succeeds', async function () { - await backups.del(boxBackup.id); - const result = await backups.get(boxBackup.id); + await backupListing.del(boxBackup.id); + const result = await backupListing.get(boxBackup.id); expect(result).to.be(null); }); it('add app backup succeeds', async function () { - appBackup.id = await backups.add(appBackup); + appBackup.id = await backupListing.add(appBackup); }); it('get app backup succeeds', async function () { - const result = await backups.get(appBackup.id); + const result = await backupListing.get(appBackup.id); delete result.creationTime; expect(result).to.eql(appBackup); }); it('getByIdentifierAndStatePaged succeeds', async function () { - const results = await backups.getByIdentifierAndStatePaged(appBackup.identifier, backups.BACKUP_STATE_CREATING, 1, 5); + const results = await backupListing.getByIdentifierAndStatePaged(appBackup.identifier, backupListing.BACKUP_STATE_CREATING, 1, 5); expect(results.length).to.be(1); delete results[0].creationTime; expect(results[0]).to.eql(appBackup); }); it('delete app backup succeeds', async function () { - await backups.del(appBackup.id); - const result = await backups.get(appBackup.id); + await backupListing.del(appBackup.id); + const result = await backupListing.get(appBackup.id); expect(result).to.be(null); }); }); diff --git a/src/test/backuptask-test.js b/src/test/backuptask-test.js index 70fa61b48..11bc53770 100644 --- a/src/test/backuptask-test.js +++ b/src/test/backuptask-test.js @@ -6,7 +6,8 @@ 'use strict'; -const backups = require('../backups.js'), +const backupListing = require('../backuplisting.js'), + backups = require('../backups.js'), common = require('./common.js'), expect = require('expect.js'), fs = require('fs'), @@ -48,7 +49,7 @@ describe('backuptask', function () { if (p.error) throw new Error(`backup failed: taskId: ${taskId} ${p.error.message}`); if (!p.result) throw new Error('backup has no result:' + p); - const result = await backups.getByIdentifierAndStatePaged(backups.BACKUP_IDENTIFIER_BOX, backups.BACKUP_STATE_NORMAL, 1, 1); + const result = await backupListing.getByIdentifierAndStatePaged(backupListing.BACKUP_IDENTIFIER_BOX, backupListing.BACKUP_STATE_NORMAL, 1, 1); if (result.length !== 1) throw new Error('result is not of length 1');