diff --git a/src/apps.js b/src/apps.js index 5dfd7fb7c..306746e0f 100644 --- a/src/apps.js +++ b/src/apps.js @@ -1491,55 +1491,53 @@ function repair(app, data, auditSource, callback) { }); } -function restore(app, backupId, auditSource, callback) { +async function restore(app, backupId, auditSource) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof backupId, 'string'); assert.strictEqual(typeof auditSource, 'object'); - assert.strictEqual(typeof callback, 'function'); const appId = app.id; let error = checkAppState(app, exports.ISTATE_PENDING_RESTORE); - if (error) return callback(error); + if (error) throw error; // for empty or null backupId, use existing manifest to mimic a reinstall - var func = backupId ? backups.get.bind(null, backupId) : function (next) { return next(null, { manifest: app.manifest }); }; + const backupInfo = backupId ? await backups.get(backupId) : { manifest: app.manifest }; - func(function (error, backupInfo) { - if (error) return callback(error); + if (!backupInfo.manifest) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Could not get restore manifest'); + if (backupInfo.encryptionVersion === 1) throw new BoxError(BoxError.BAD_FIELD, 'This encrypted backup was created with an older Cloudron version and has to be restored using the CLI tool'); - if (!backupInfo.manifest) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Could not get restore manifest')); - if (backupInfo.encryptionVersion === 1) return callback(new BoxError(BoxError.BAD_FIELD, 'This encrypted backup was created with an older Cloudron version and has to be restored using the CLI tool')); + // re-validate because this new box version may not accept old configs + error = checkManifestConstraints(backupInfo.manifest); + if (error) throw error; - // re-validate because this new box version may not accept old configs - error = checkManifestConstraints(backupInfo.manifest); - if (error) return callback(error); + let values = { manifest: backupInfo.manifest }; + if (!hasMailAddon(backupInfo.manifest)) { // clear if restore removed addon + values.mailboxName = values.mailboxDomain = null; + } else if (!app.mailboxName || app.mailboxName.endsWith('.app')) { // allocate since restore added the addon + values.mailboxName = mailboxNameForLocation(app.location, backupInfo.manifest); + values.mailboxDomain = app.domain; + } - let values = { manifest: backupInfo.manifest }; - if (!hasMailAddon(backupInfo.manifest)) { // clear if restore removed addon - values.mailboxName = values.mailboxDomain = null; - } else if (!app.mailboxName || app.mailboxName.endsWith('.app')) { // allocate since restore added the addon - values.mailboxName = mailboxNameForLocation(app.location, backupInfo.manifest); - values.mailboxDomain = app.domain; - } + const restoreConfig = { backupId, backupFormat: backupInfo.format }; - const restoreConfig = { backupId, backupFormat: backupInfo.format }; + const task = { + args: { + restoreConfig, + oldManifest: app.manifest, + skipDnsSetup: !!backupId, // if this is a restore, just skip dns setup. only re-installs should setup dns + overwriteDns: true + }, + values + }; - const task = { - args: { - restoreConfig, - oldManifest: app.manifest, - skipDnsSetup: !!backupId, // if this is a restore, just skip dns setup. only re-installs should setup dns - overwriteDns: true - }, - values - }; + return new Promise((resolve, reject) => { addTask(appId, exports.ISTATE_PENDING_RESTORE, task, function (error, result) { - if (error) return callback(error); + if (error) return reject(error); eventlog.add(eventlog.ACTION_APP_RESTORE, auditSource, { app: app, backupId: backupInfo.id, fromManifest: app.manifest, toManifest: backupInfo.manifest, taskId: result.taskId }); - callback(null, { taskId: result.taskId }); + resolve({ taskId: result.taskId }); }); }); } @@ -1656,9 +1654,13 @@ function clone(app, data, user, auditSource, callback) { assert.strictEqual(typeof domain, 'string'); assert.strictEqual(typeof portBindings, 'object'); - backups.get(backupId, function (error, backupInfo) { + const locations = [{ subdomain: location, domain, type: 'primary' }]; + validateLocations(locations, async function (error, domainObjectMap) { if (error) return callback(error); + const [backupsError, backupInfo] = await safe(backups.get(backupId)); + if (backupsError) return callback(backupsError); + if (!backupInfo.manifest) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Could not get restore config')); if (backupInfo.encryptionVersion === 1) return callback(new BoxError(BoxError.BAD_FIELD, 'This encrypted backup was created with an older Cloudron version and cannot be cloned')); @@ -1675,62 +1677,57 @@ function clone(app, data, user, auditSource, callback) { let mailboxName = hasMailAddon(manifest) ? mailboxNameForLocation(location, manifest) : null; let mailboxDomain = hasMailAddon(manifest) ? domain : null; - const locations = [{ subdomain: location, domain, type: 'primary' }]; - validateLocations(locations, function (error, domainObjectMap) { + const newAppId = uuid.v4(); + + appdb.getIcons(app.id, function (error, icons) { if (error) return callback(error); - const newAppId = uuid.v4(); + const data = { + installationState: exports.ISTATE_PENDING_CLONE, + runState: exports.RSTATE_RUNNING, + memoryLimit: app.memoryLimit, + cpuShares: app.cpuShares, + accessRestriction: app.accessRestriction, + sso: !!app.sso, + mailboxName: mailboxName, + mailboxDomain: mailboxDomain, + enableBackup: app.enableBackup, + reverseProxyConfig: app.reverseProxyConfig, + env: app.env, + alternateDomains: [], + aliasDomains: [], + servicesConfig: app.servicesConfig, + label: app.label ? `${app.label}-clone` : '', + tags: app.tags, + enableAutomaticUpdate: app.enableAutomaticUpdate, + icon: icons.icon, + enableMailbox: app.enableMailbox + }; - appdb.getIcons(app.id, function (error, icons) { + appdb.add(newAppId, appStoreId, manifest, location, domain, translatePortBindings(portBindings, manifest), data, function (error) { + if (error && error.reason === BoxError.ALREADY_EXISTS) return callback(getDuplicateErrorDetails(error.message, locations, domainObjectMap, portBindings)); if (error) return callback(error); - const data = { - installationState: exports.ISTATE_PENDING_CLONE, - runState: exports.RSTATE_RUNNING, - memoryLimit: app.memoryLimit, - cpuShares: app.cpuShares, - accessRestriction: app.accessRestriction, - sso: !!app.sso, - mailboxName: mailboxName, - mailboxDomain: mailboxDomain, - enableBackup: app.enableBackup, - reverseProxyConfig: app.reverseProxyConfig, - env: app.env, - alternateDomains: [], - aliasDomains: [], - servicesConfig: app.servicesConfig, - label: app.label ? `${app.label}-clone` : '', - tags: app.tags, - enableAutomaticUpdate: app.enableAutomaticUpdate, - icon: icons.icon, - enableMailbox: app.enableMailbox - }; - - appdb.add(newAppId, appStoreId, manifest, location, domain, translatePortBindings(portBindings, manifest), data, function (error) { - if (error && error.reason === BoxError.ALREADY_EXISTS) return callback(getDuplicateErrorDetails(error.message, locations, domainObjectMap, portBindings)); + purchaseApp({ appId: newAppId, appstoreId: app.appStoreId, manifestId: manifest.id || 'customapp' }, function (error) { if (error) return callback(error); - purchaseApp({ appId: newAppId, appstoreId: app.appStoreId, manifestId: manifest.id || 'customapp' }, function (error) { + const restoreConfig = { backupId: backupId, backupFormat: backupInfo.format }; + const task = { + args: { restoreConfig, overwriteDns, skipDnsSetup, oldManifest: null }, + values: {}, + requiredState: exports.ISTATE_PENDING_CLONE + }; + addTask(newAppId, exports.ISTATE_PENDING_CLONE, task, function (error, result) { if (error) return callback(error); - const restoreConfig = { backupId: backupId, backupFormat: backupInfo.format }; - const task = { - args: { restoreConfig, overwriteDns, skipDnsSetup, oldManifest: null }, - values: {}, - requiredState: exports.ISTATE_PENDING_CLONE - }; - addTask(newAppId, exports.ISTATE_PENDING_CLONE, task, function (error, result) { - if (error) return callback(error); + const newApp = _.extend({}, _.omit(data, 'icon'), { appStoreId, manifest, location, domain, portBindings }); + newApp.fqdn = domains.fqdn(newApp.location, domainObjectMap[newApp.domain]); + newApp.alternateDomains.forEach(function (ad) { ad.fqdn = domains.fqdn(ad.subdomain, domainObjectMap[ad.domain]); }); + newApp.aliasDomains.forEach(function (ad) { ad.fqdn = domains.fqdn(ad.subdomain, domainObjectMap[ad.domain]); }); - const newApp = _.extend({}, _.omit(data, 'icon'), { appStoreId, manifest, location, domain, portBindings }); - newApp.fqdn = domains.fqdn(newApp.location, domainObjectMap[newApp.domain]); - newApp.alternateDomains.forEach(function (ad) { ad.fqdn = domains.fqdn(ad.subdomain, domainObjectMap[ad.domain]); }); - newApp.aliasDomains.forEach(function (ad) { ad.fqdn = domains.fqdn(ad.subdomain, domainObjectMap[ad.domain]); }); + eventlog.add(eventlog.ACTION_APP_CLONE, auditSource, { appId: newAppId, oldAppId: appId, backupId: backupId, oldApp: app, newApp: newApp, taskId: result.taskId }); - eventlog.add(eventlog.ACTION_APP_CLONE, auditSource, { appId: newAppId, oldAppId: appId, backupId: backupId, oldApp: app, newApp: newApp, taskId: result.taskId }); - - callback(null, { id: newAppId, taskId: result.taskId }); - }); + callback(null, { id: newAppId, taskId: result.taskId }); }); }); }); @@ -1976,59 +1973,54 @@ function backup(app, callback) { }); } -function listBackups(app, page, perPage, callback) { +async function listBackups(app, page, perPage) { assert.strictEqual(typeof app, 'object'); assert(typeof page === 'number' && page > 0); assert(typeof perPage === 'number' && perPage > 0); - assert.strictEqual(typeof callback, 'function'); - backups.getByIdentifierAndStatePaged(app.id, backups.BACKUP_STATE_NORMAL, page, perPage, function (error, results) { - if (error) return callback(error); - - callback(null, results); - }); + return await backups.getByIdentifierAndStatePaged(app.id, backups.BACKUP_STATE_NORMAL, page, perPage); } function restoreInstalledApps(options, callback) { assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof callback, 'function'); - getAll(function (error, apps) { + const addTaskAsync = util.promisify(addTask); + + getAll(async function (error, apps) { if (error) return callback(error); apps = apps.filter(app => app.installationState !== exports.ISTATE_ERROR); // remove errored apps. let them be 'repaired' by hand apps = apps.filter(app => app.installationState !== exports.ISTATE_PENDING_RESTORE); // safeguard against tasks being created non-stop if we crash on startup - async.eachSeries(apps, function (app, iteratorDone) { - backups.getByIdentifierAndStatePaged(app.id, backups.BACKUP_STATE_NORMAL, 1, 1, function (error, results) { - let installationState, restoreConfig, oldManifest; - if (!error && results.length) { - installationState = exports.ISTATE_PENDING_RESTORE; - restoreConfig = { backupId: results[0].id, backupFormat: results[0].format }; - oldManifest = app.manifest; - } else { - installationState = exports.ISTATE_PENDING_INSTALL; - restoreConfig = null; - oldManifest = null; - } + for (const app of apps) { + const [error, results] = await safe(backups.getByIdentifierAndStatePaged(app.id, backups.BACKUP_STATE_NORMAL, 1, 1)); + let installationState, restoreConfig, oldManifest; + if (!error && results.length) { + installationState = exports.ISTATE_PENDING_RESTORE; + restoreConfig = { backupId: results[0].id, backupFormat: results[0].format }; + oldManifest = app.manifest; + } else { + installationState = exports.ISTATE_PENDING_INSTALL; + restoreConfig = null; + oldManifest = null; + } - const task = { - args: { restoreConfig, skipDnsSetup: options.skipDnsSetup, overwriteDns: true, oldManifest }, - values: {}, - scheduleNow: false, // task will be scheduled by autoRestartTasks when platform is ready - requireNullTaskId: false // ignore existing stale taskId - }; + const task = { + args: { restoreConfig, skipDnsSetup: options.skipDnsSetup, overwriteDns: true, oldManifest }, + values: {}, + scheduleNow: false, // task will be scheduled by autoRestartTasks when platform is ready + requireNullTaskId: false // ignore existing stale taskId + }; - debug(`restoreInstalledApps: marking ${app.fqdn} for restore using restore config ${JSON.stringify(restoreConfig)}`); + debug(`restoreInstalledApps: marking ${app.fqdn} for restore using restore config ${JSON.stringify(restoreConfig)}`); - addTask(app.id, installationState, task, function (error, result) { - if (error) debug(`restoreInstalledApps: error marking ${app.fqdn} for restore: ${JSON.stringify(error)}`); - else debug(`restoreInstalledApps: marked ${app.id} for restore with taskId ${result.taskId}`); + const [addTaskError, result] = await safe(addTaskAsync(app.id, installationState, task)); + if (addTaskError) debug(`restoreInstalledApps: error marking ${app.fqdn} for restore: ${JSON.stringify(addTaskError)}`); + else debug(`restoreInstalledApps: marked ${app.id} for restore with taskId ${result.taskId}`); + } - iteratorDone(); // ignore error - }); - }); - }, callback); + callback(null); }); } diff --git a/src/apptask.js b/src/apptask.js index fec6ebe7d..47e7f6100 100644 --- a/src/apptask.js +++ b/src/apptask.js @@ -16,7 +16,7 @@ const appdb = require('./appdb.js'), apps = require('./apps.js'), assert = require('assert'), async = require('async'), - backups = require('./backups.js'), + backuptask = require('./backuptask.js'), BoxError = require('./boxerror.js'), collectd = require('./collectd.js'), constants = require('./constants.js'), @@ -467,7 +467,7 @@ function install(app, args, progressCallback, callback) { progressCallback.bind(null, { percent: 65, message: 'Download backup and restoring addons' }), services.setupAddons.bind(null, app, app.manifest.addons), services.clearAddons.bind(null, app, app.manifest.addons), - backups.downloadApp.bind(null, app, restoreConfig, (progress) => { + backuptask.downloadApp.bind(null, app, restoreConfig, (progress) => { progressCallback({ percent: 65, message: progress.message }); }), (done) => { if (app.installationState === apps.ISTATE_PENDING_IMPORT) apps.restoreConfig(app, done); else done(); }, @@ -513,7 +513,7 @@ function backup(app, args, progressCallback, callback) { async.series([ progressCallback.bind(null, { percent: 10, message: 'Backing up' }), - backups.backupApp.bind(null, app, { snapshotOnly: !!args.snapshotOnly }, (progress) => { + backuptask.backupApp.bind(null, app, { snapshotOnly: !!args.snapshotOnly }, (progress) => { progressCallback({ percent: 30, message: progress.message }); }), @@ -746,7 +746,7 @@ function update(app, args, progressCallback, callback) { async.series([ progressCallback.bind(null, { percent: 15, message: 'Backing up app' }), // preserve update backups for 3 weeks - backups.backupApp.bind(null, app, { preserveSecs: 3*7*24*60*60 }, (progress) => { + backuptask.backupApp.bind(null, app, { preserveSecs: 3*7*24*60*60 }, (progress) => { progressCallback({ percent: 15, message: `Backup - ${progress.message}` }); }) ], function (error) { diff --git a/src/backupcleaner.js b/src/backupcleaner.js new file mode 100644 index 000000000..25eff8756 --- /dev/null +++ b/src/backupcleaner.js @@ -0,0 +1,300 @@ +'use strict'; + +exports = module.exports = { + run, + + _applyBackupRetentionPolicy: applyBackupRetentionPolicy +}; + +const apps = require('./apps.js'), + assert = require('assert'), + async = require('async'), + backups = require('./backups.js'), + BoxError = require('./boxerror.js'), + constants = require('./constants.js'), + debug = require('debug')('box:backupcleaner'), + moment = require('moment'), + path = require('path'), + paths = require('./paths.js'), + safe = require('safetydance'), + settings = require('./settings.js'), + storage = require('./storage.js'), + util = require('util'), + _ = require('underscore'); + +function applyBackupRetentionPolicy(backups, policy, referencedBackupIds) { + assert(Array.isArray(backups)); + assert.strictEqual(typeof policy, 'object'); + assert(Array.isArray(referencedBackupIds)); + + const now = new Date(); + + for (const backup of backups) { + if (backup.state === backups.BACKUP_STATE_ERROR) { + backup.discardReason = 'error'; + } else if (backup.state === backups.BACKUP_STATE_CREATING) { + if ((now - backup.creationTime) < 48*60*60*1000) backup.keepReason = 'creating'; + else backup.discardReason = 'creating-too-long'; + } else if (referencedBackupIds.includes(backup.id)) { + backup.keepReason = 'reference'; + } else if ((now - backup.creationTime) < (backup.preserveSecs * 1000)) { + backup.keepReason = 'preserveSecs'; + } else if ((now - backup.creationTime < policy.keepWithinSecs * 1000) || policy.keepWithinSecs < 0) { + backup.keepReason = 'keepWithinSecs'; + } + } + + const KEEP_FORMATS = { + keepDaily: 'Y-M-D', + keepWeekly: 'Y-W', + keepMonthly: 'Y-M', + keepYearly: 'Y' + }; + + for (const format of [ 'keepDaily', 'keepWeekly', 'keepMonthly', 'keepYearly' ]) { + if (!(format in policy)) continue; + + const n = policy[format]; // we want to keep "n" backups of format + if (!n) continue; // disabled rule + + let lastPeriod = null, keptSoFar = 0; + for (const backup of backups) { + if (backup.discardReason) continue; // already discarded for some reason + if (backup.keepReason && backup.keepReason !== 'reference') continue; // kept for some other reason + const period = moment(backup.creationTime).format(KEEP_FORMATS[format]); + if (period === lastPeriod) continue; // already kept for this period + + lastPeriod = period; + backup.keepReason = backup.keepReason ? `${backup.keepReason}+${format}` : format; + if (++keptSoFar === n) break; + } + } + + if (policy.keepLatest) { + let latestNormalBackup = backups.find(b => b.state === backups.BACKUP_STATE_NORMAL); + if (latestNormalBackup && !latestNormalBackup.keepReason) latestNormalBackup.keepReason = 'latest'; + } + + for (const backup of backups) { + debug(`applyBackupRetentionPolicy: ${backup.id} ${backup.type} ${backup.keepReason || backup.discardReason || 'unprocessed'}`); + } +} + +async function cleanupBackup(backupConfig, backup, progressCallback) { + assert.strictEqual(typeof backupConfig, 'object'); + assert.strictEqual(typeof backup, 'object'); + assert.strictEqual(typeof progressCallback, 'function'); + + const backupFilePath = storage.getBackupFilePath(backupConfig, backup.id, backup.format); + + return new Promise((resolve) => { + function done(error) { + if (error) { + debug('cleanupBackup: error removing backup %j : %s', backup, error.message); + return resolve(); + } + + // prune empty directory if possible + storage.api(backupConfig.provider).remove(backupConfig, path.dirname(backupFilePath), async function (error) { + if (error) debug('cleanupBackup: unable to prune backup directory %s : %s', path.dirname(backupFilePath), error.message); + + const [delError] = await safe(backups.del(backup.id)); + if (delError) debug('cleanupBackup: error removing from database', delError); + else debug('cleanupBackup: removed %s', backup.id); + + resolve(); + }); + } + + if (backup.format ==='tgz') { + progressCallback({ message: `${backup.id}: Removing ${backupFilePath}`}); + storage.api(backupConfig.provider).remove(backupConfig, backupFilePath, done); + } else { + const events = storage.api(backupConfig.provider).removeDir(backupConfig, backupFilePath); + events.on('progress', (message) => progressCallback({ message: `${backup.id}: ${message}` })); + events.on('done', done); + } + }); +} + +function cleanupAppBackups(backupConfig, referencedAppBackupIds, progressCallback, callback) { + assert.strictEqual(typeof backupConfig, 'object'); + assert(Array.isArray(referencedAppBackupIds)); + assert.strictEqual(typeof progressCallback, 'function'); + assert.strictEqual(typeof callback, 'function'); + + let removedAppBackupIds = []; + + apps.getAll(async function (error, allApps) { + if (error) return callback(error); + + const allAppIds = allApps.map(a => a.id); + + const [getError, appBackups] = await safe(backups.getByTypePaged(backups.BACKUP_TYPE_APP, 1, 1000)); + if (getError) return callback(getError); + + // collate the backups by app id. note that the app could already have been uninstalled + let appBackupsById = {}; + for (const appBackup of appBackups) { + if (!appBackupsById[appBackup.identifier]) appBackupsById[appBackup.identifier] = []; + appBackupsById[appBackup.identifier].push(appBackup); + } + + // apply backup policy per app. keep latest backup only for existing apps + let appBackupsToRemove = []; + for (const appId of Object.keys(appBackupsById)) { + applyBackupRetentionPolicy(appBackupsById[appId], _.extend({ keepLatest: allAppIds.includes(appId) }, backupConfig.retentionPolicy), referencedAppBackupIds); + appBackupsToRemove = appBackupsToRemove.concat(appBackupsById[appId].filter(b => !b.keepReason)); + } + + for (const appBackup of appBackupsToRemove) { + progressCallback({ message: `Removing app backup (${appBackup.identifier}): ${appBackup.id}`}); + removedAppBackupIds.push(appBackup.id); + await cleanupBackup(backupConfig, appBackup, progressCallback); // never errors + } + + debug('cleanupAppBackups: done'); + + callback(null, removedAppBackupIds); + }); +} + +async function cleanupBoxBackups(backupConfig, progressCallback) { + assert.strictEqual(typeof backupConfig, 'object'); + assert.strictEqual(typeof progressCallback, 'function'); + + let referencedAppBackupIds = [], removedBoxBackupIds = []; + + const boxBackups = await backups.getByTypePaged(backups.BACKUP_TYPE_BOX, 1, 1000); + + applyBackupRetentionPolicy(boxBackups, _.extend({ keepLatest: true }, backupConfig.retentionPolicy), [] /* references */); + + for (const boxBackup of boxBackups) { + if (boxBackup.keepReason) { + referencedAppBackupIds = referencedAppBackupIds.concat(boxBackup.dependsOn); + continue; + } + + progressCallback({ message: `Removing box backup ${boxBackup.id}`}); + + removedBoxBackupIds.push(boxBackup.id); + await cleanupBackup(backupConfig, boxBackup, progressCallback); + } + + debug('cleanupBoxBackups: done'); + + return { removedBoxBackupIds, referencedAppBackupIds }; +} + +async function cleanupMissingBackups(backupConfig, progressCallback) { + assert.strictEqual(typeof backupConfig, 'object'); + assert.strictEqual(typeof progressCallback, 'function'); + + const perPage = 1000; + let missingBackupIds = []; + const backupExists = util.promisify(storage.api(backupConfig.provider).exists); + + if (constants.TEST) return missingBackupIds; + + for (let page = 0, result = []; result.length < perPage; page++) { + result = await backups.list(page, perPage); + + for (const backup of result) { + let backupFilePath = storage.getBackupFilePath(backupConfig, backup.id, backup.format); + if (backup.format === 'rsync') backupFilePath = backupFilePath + '/'; // add trailing slash to indicate directory + + const [existsError, exists] = await safe(backupExists(backupConfig, backupFilePath)); + if (existsError || exists) continue; + + progressCallback({ message: `Removing missing backup ${backup.id}`}); + + const [delError] = await safe(backups.del(backup.id)); + if (delError) debug(`cleanupBackup: error removing ${backup.id} from database`, delError); + + missingBackupIds.push(backup.id); + } + } + + return missingBackupIds; +} + +// removes the snapshots of apps that have been uninstalled +function cleanupSnapshots(backupConfig, callback) { + assert.strictEqual(typeof backupConfig, 'object'); + assert.strictEqual(typeof callback, 'function'); + + var contents = safe.fs.readFileSync(paths.SNAPSHOT_INFO_FILE, 'utf8'); + var info = safe.JSON.parse(contents); + if (!info) return callback(); + + delete info.box; + async.eachSeries(Object.keys(info), function (appId, iteratorDone) { + apps.get(appId, function (error /*, app */) { + if (!error || error.reason !== BoxError.NOT_FOUND) return iteratorDone(); + + function done(/* ignoredError */) { + safe.fs.unlinkSync(path.join(paths.BACKUP_INFO_DIR, `${appId}.sync.cache`)); + safe.fs.unlinkSync(path.join(paths.BACKUP_INFO_DIR, `${appId}.sync.cache.new`)); + + backups.setSnapshotInfo(appId, null, function (/* ignoredError */) { + debug('cleanupSnapshots: cleaned up snapshot of app id %s', appId); + + iteratorDone(); + }); + } + + if (info[appId].format ==='tgz') { + storage.api(backupConfig.provider).remove(backupConfig, storage.getBackupFilePath(backupConfig, `snapshot/app_${appId}`, info[appId].format), done); + } else { + var events = storage.api(backupConfig.provider).removeDir(backupConfig, storage.getBackupFilePath(backupConfig, `snapshot/app_${appId}`, info[appId].format)); + events.on('progress', function (detail) { debug(`cleanupSnapshots: ${detail}`); }); + events.on('done', done); + } + }); + }, function () { + debug('cleanupSnapshots: done'); + + callback(); + }); +} + +function run(progressCallback, callback) { + assert.strictEqual(typeof progressCallback, 'function'); + assert.strictEqual(typeof callback, 'function'); + + settings.getBackupConfig(async function (error, backupConfig) { + if (error) return callback(error); + + if (backupConfig.retentionPolicy.keepWithinSecs < 0) { + debug('cleanup: keeping all backups'); + return callback(null, {}); + } + + progressCallback({ percent: 10, message: 'Cleaning box backups' }); + + const [cleanupBoxError, removedBoxBackups ] = await safe(cleanupBoxBackups(backupConfig, progressCallback)); + if (cleanupBoxError) return callback(cleanupBoxError); + + const { removedBoxBackupIds, referencedAppBackupIds } = removedBoxBackups; + + progressCallback({ percent: 40, message: 'Cleaning app backups' }); + + cleanupAppBackups(backupConfig, referencedAppBackupIds, progressCallback, function (error, removedAppBackupIds) { + if (error) return callback(error); + + progressCallback({ percent: 70, message: 'Cleaning missing backups' }); + + cleanupMissingBackups(backupConfig, progressCallback, function (error, missingBackupIds) { + if (error) return callback(error); + + progressCallback({ percent: 90, message: 'Cleaning snapshots' }); + + cleanupSnapshots(backupConfig, function (error) { + if (error) return callback(error); + + callback(null, { removedBoxBackupIds, removedAppBackupIds, missingBackupIds }); + }); + }); + }); + }); +} diff --git a/src/backupdb.js b/src/backupdb.js deleted file mode 100644 index 878122a5d..000000000 --- a/src/backupdb.js +++ /dev/null @@ -1,176 +0,0 @@ -'use strict'; - -const assert = require('assert'), - BoxError = require('./boxerror.js'), - database = require('./database.js'), - safe = require('safetydance'); - -const BACKUPS_FIELDS = [ 'id', 'identifier', 'creationTime', 'packageVersion', 'type', 'dependsOn', 'state', 'manifestJson', 'format', 'preserveSecs', 'encryptionVersion' ]; - -exports = module.exports = { - add, - - getByTypePaged, - getByIdentifierPaged, - getByIdentifierAndStatePaged, - - get, - del, - update, - list, - - _clear: clear -}; - -function postProcess(result) { - assert.strictEqual(typeof result, 'object'); - - result.dependsOn = result.dependsOn ? result.dependsOn.split(',') : [ ]; - - result.manifest = result.manifestJson ? safe.JSON.parse(result.manifestJson) : null; - delete result.manifestJson; -} - -function getByIdentifierAndStatePaged(identifier, state, page, perPage, callback) { - assert.strictEqual(typeof identifier, 'string'); - assert.strictEqual(typeof state, 'string'); - assert(typeof page === 'number' && page > 0); - assert(typeof perPage === 'number' && perPage > 0); - assert.strictEqual(typeof callback, 'function'); - - database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups WHERE identifier = ? AND state = ? ORDER BY creationTime DESC LIMIT ?,?', - [ identifier, state, (page-1)*perPage, perPage ], function (error, results) { - if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error)); - - results.forEach(function (result) { postProcess(result); }); - - callback(null, results); - }); -} - -function getByTypePaged(type, page, perPage, callback) { - assert.strictEqual(typeof type, 'string'); - assert(typeof page === 'number' && page > 0); - assert(typeof perPage === 'number' && perPage > 0); - assert.strictEqual(typeof callback, 'function'); - - database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups WHERE type = ? ORDER BY creationTime DESC LIMIT ?,?', - [ type, (page-1)*perPage, perPage ], function (error, results) { - if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error)); - - results.forEach(function (result) { postProcess(result); }); - - callback(null, results); - }); -} - -function getByIdentifierPaged(identifier, page, perPage, callback) { - assert.strictEqual(typeof identifier, 'string'); - assert(typeof page === 'number' && page > 0); - assert(typeof perPage === 'number' && perPage > 0); - assert.strictEqual(typeof callback, 'function'); - - database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups WHERE identifier = ? ORDER BY creationTime DESC LIMIT ?,?', - [ identifier, (page-1)*perPage, perPage ], function (error, results) { - if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error)); - - results.forEach(function (result) { postProcess(result); }); - - callback(null, results); - }); -} - -function list(page, perPage, callback) { - assert(typeof page === 'number' && page > 0); - assert(typeof perPage === 'number' && perPage > 0); - assert.strictEqual(typeof callback, 'function'); - - database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups ORDER BY creationTime DESC LIMIT ?,?', - [ (page-1)*perPage, perPage ], function (error, results) { - if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error)); - - results.forEach(function (result) { postProcess(result); }); - - callback(null, results); - }); -} - -function get(id, callback) { - assert.strictEqual(typeof id, 'string'); - assert.strictEqual(typeof callback, 'function'); - - database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups WHERE id = ? ORDER BY creationTime DESC', - [ id ], function (error, result) { - if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error)); - if (result.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'Backup not found')); - - postProcess(result[0]); - - callback(null, result[0]); - }); -} - -function add(id, data, callback) { - assert(data && typeof data === 'object'); - assert.strictEqual(typeof id, 'string'); - assert(data.encryptionVersion === null || typeof data.encryptionVersion === 'number'); - assert.strictEqual(typeof data.packageVersion, 'string'); - assert.strictEqual(typeof data.type, 'string'); - assert.strictEqual(typeof data.identifier, 'string'); - assert.strictEqual(typeof data.state, 'string'); - assert(Array.isArray(data.dependsOn)); - assert.strictEqual(typeof data.manifest, 'object'); - assert.strictEqual(typeof data.format, 'string'); - assert.strictEqual(typeof callback, 'function'); - - var creationTime = data.creationTime || new Date(); // allow tests to set the time - var manifestJson = JSON.stringify(data.manifest); - - database.query('INSERT INTO backups (id, identifier, encryptionVersion, packageVersion, type, creationTime, state, dependsOn, manifestJson, format) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', - [ id, data.identifier, data.encryptionVersion, data.packageVersion, data.type, creationTime, data.state, data.dependsOn.join(','), manifestJson, data.format ], - function (error) { - if (error && error.code === 'ER_DUP_ENTRY') return callback(new BoxError(BoxError.ALREADY_EXISTS)); - if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error)); - - callback(null); - }); -} - -function update(id, backup, callback) { - assert.strictEqual(typeof id, 'string'); - assert.strictEqual(typeof backup, 'object'); - assert.strictEqual(typeof callback, 'function'); - - var fields = [ ], values = [ ]; - for (var p in backup) { - fields.push(p + ' = ?'); - values.push(backup[p]); - } - values.push(id); - - database.query('UPDATE backups SET ' + fields.join(', ') + ' WHERE id = ?', values, function (error) { - if (error && error.reason === BoxError.NOT_FOUND) return callback(new BoxError(BoxError.NOT_FOUND, 'Backup not found')); - if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error)); - - callback(null); - }); -} - -function clear(callback) { - assert.strictEqual(typeof callback, 'function'); - - database.query('TRUNCATE TABLE backups', [], function (error) { - if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error)); - callback(null); - }); -} - -function del(id, callback) { - assert.strictEqual(typeof id, 'string'); - assert.strictEqual(typeof callback, 'function'); - - database.query('DELETE FROM backups WHERE id=?', [ id ], function (error) { - if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error)); - callback(null); - }); -} diff --git a/src/backups.js b/src/backups.js index e6658cf12..66ced9c87 100644 --- a/src/backups.js +++ b/src/backups.js @@ -1,26 +1,17 @@ 'use strict'; exports = module.exports = { - testConfig, - testProviderConfig, - - getByIdentifierAndStatePaged, - get, + getByIdentifierAndStatePaged, + getByTypePaged, + add, + update, + list, + del, startBackupTask, - restore, - - backupApp, - downloadApp, - - backupBoxAndApps, - - upload, - startCleanupTask, - cleanup, cleanupCacheFilesSync, injectPrivateFields, @@ -29,7 +20,12 @@ exports = module.exports = { configureCollectd, generateEncryptionKeysSync, - isMountProvider, + + getSnapshotInfo, + setSnapshotInfo, + + testConfig, + testProviderConfig, BACKUP_IDENTIFIER_BOX: 'box', @@ -39,85 +35,39 @@ exports = module.exports = { BACKUP_STATE_NORMAL: 'normal', // should rename to created to avoid listing in UI? BACKUP_STATE_CREATING: 'creating', BACKUP_STATE_ERROR: 'error', - - // for testing - _getBackupFilePath: getBackupFilePath, - _restoreFsMetadata: restoreFsMetadata, - _saveFsMetadata: saveFsMetadata, - _applyBackupRetentionPolicy: applyBackupRetentionPolicy }; -const apps = require('./apps.js'), - async = require('async'), - assert = require('assert'), - backupdb = require('./backupdb.js'), +const assert = require('assert'), BoxError = require('./boxerror.js'), collectd = require('./collectd.js'), constants = require('./constants.js'), CronJob = require('cron').CronJob, crypto = require('crypto'), database = require('./database.js'), - DataLayout = require('./datalayout.js'), - debug = require('debug')('box:backups'), ejs = require('ejs'), eventlog = require('./eventlog.js'), fs = require('fs'), locker = require('./locker.js'), - moment = require('moment'), - once = require('once'), path = require('path'), paths = require('./paths.js'), - progressStream = require('progress-stream'), safe = require('safetydance'), - shell = require('./shell.js'), - services = require('./services.js'), settings = require('./settings.js'), - syncer = require('./syncer.js'), - tar = require('tar-fs'), - tasks = require('./tasks.js'), - TransformStream = require('stream').Transform, - util = require('util'), - zlib = require('zlib'), - _ = require('underscore'); + storage = require('./storage.js'), + tasks = require('./tasks.js'); -const BACKUP_UPLOAD_CMD = path.join(__dirname, 'scripts/backupupload.js'); const COLLECTD_CONFIG_EJS = fs.readFileSync(__dirname + '/collectd/cloudron-backup.ejs', { encoding: 'utf8' }); -function debugApp(app) { - assert(typeof app === 'object'); +const BACKUPS_FIELDS = [ 'id', 'identifier', 'creationTime', 'packageVersion', 'type', 'dependsOn', 'state', 'manifestJson', 'format', 'preserveSecs', 'encryptionVersion' ]; - debug(app.fqdn + ' ' + util.format.apply(util, Array.prototype.slice.call(arguments, 1))); -} +function postProcess(result) { + assert.strictEqual(typeof result, 'object'); -// choose which storage backend we use for test purpose we use s3 -function api(provider) { - switch (provider) { - case 'nfs': return require('./storage/filesystem.js'); - case 'cifs': return require('./storage/filesystem.js'); - case 'sshfs': return require('./storage/filesystem.js'); - case 'mountpoint': return require('./storage/filesystem.js'); - case 'ext4': return require('./storage/filesystem.js'); - case 's3': return require('./storage/s3.js'); - case 'gcs': return require('./storage/gcs.js'); - case 'filesystem': return require('./storage/filesystem.js'); - case 'minio': return require('./storage/s3.js'); - case 's3-v4-compat': return require('./storage/s3.js'); - case 'digitalocean-spaces': return require('./storage/s3.js'); - case 'exoscale-sos': return require('./storage/s3.js'); - case 'wasabi': return require('./storage/s3.js'); - case 'scaleway-objectstorage': return require('./storage/s3.js'); - case 'backblaze-b2': return require('./storage/s3.js'); - case 'linode-objectstorage': return require('./storage/s3.js'); - case 'ovh-objectstorage': return require('./storage/s3.js'); - case 'ionos-objectstorage': return require('./storage/s3.js'); - case 'vultr-objectstorage': return require('./storage/s3.js'); - case 'noop': return require('./storage/noop.js'); - default: return null; - } -} + result.dependsOn = result.dependsOn ? result.dependsOn.split(',') : [ ]; -function isMountProvider(provider) { - return provider === 'sshfs' || provider === 'cifs' || provider === 'nfs' || provider === 'ext4'; + result.manifest = result.manifestJson ? safe.JSON.parse(result.manifestJson) : null; + delete result.manifestJson; + + return result; } function injectPrivateFields(newConfig, currentConfig) { @@ -129,7 +79,7 @@ function injectPrivateFields(newConfig, currentConfig) { } else { newConfig.encryption = null; } - if (newConfig.provider === currentConfig.provider) api(newConfig.provider).injectPrivateFields(newConfig, currentConfig); + if (newConfig.provider === currentConfig.provider) storage.api(newConfig.provider).injectPrivateFields(newConfig, currentConfig); } function removePrivateFields(backupConfig) { @@ -138,47 +88,7 @@ function removePrivateFields(backupConfig) { delete backupConfig.encryption; backupConfig.password = constants.SECRET_PLACEHOLDER; } - return api(backupConfig.provider).removePrivateFields(backupConfig); -} - -function testConfig(backupConfig, callback) { - assert.strictEqual(typeof backupConfig, 'object'); - assert.strictEqual(typeof callback, 'function'); - - const func = api(backupConfig.provider); - if (!func) return callback(new BoxError(BoxError.BAD_FIELD, 'unknown storage provider', { field: 'provider' })); - - if (backupConfig.format !== 'tgz' && backupConfig.format !== 'rsync') return callback(new BoxError(BoxError.BAD_FIELD, 'unknown format', { field: 'format' })); - - const job = safe.safeCall(function () { return new CronJob(backupConfig.schedulePattern); }); - if (!job) return callback(new BoxError(BoxError.BAD_FIELD, 'Invalid schedule pattern', { field: 'schedulePattern' })); - - if ('password' in backupConfig) { - if (typeof backupConfig.password !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'password must be a string', { field: 'password' })); - if (backupConfig.password.length < 8) return callback(new BoxError(BoxError.BAD_FIELD, 'password must be atleast 8 characters', { field: 'password' })); - } - - const policy = backupConfig.retentionPolicy; - if (!policy) return callback(new BoxError(BoxError.BAD_FIELD, 'retentionPolicy is required', { field: 'retentionPolicy' })); - if (!['keepWithinSecs','keepDaily','keepWeekly','keepMonthly','keepYearly'].find(k => !!policy[k])) return callback(new BoxError(BoxError.BAD_FIELD, 'properties missing', { field: 'retentionPolicy' })); - if ('keepWithinSecs' in policy && typeof policy.keepWithinSecs !== 'number') return callback(new BoxError(BoxError.BAD_FIELD, 'keepWithinSecs must be a number', { field: 'retentionPolicy' })); - if ('keepDaily' in policy && typeof policy.keepDaily !== 'number') return callback(new BoxError(BoxError.BAD_FIELD, 'keepDaily must be a number', { field: 'retentionPolicy' })); - if ('keepWeekly' in policy && typeof policy.keepWeekly !== 'number') return callback(new BoxError(BoxError.BAD_FIELD, 'keepWeekly must be a number', { field: 'retentionPolicy' })); - if ('keepMonthly' in policy && typeof policy.keepMonthly !== 'number') return callback(new BoxError(BoxError.BAD_FIELD, 'keepMonthly must be a number', { field: 'retentionPolicy' })); - if ('keepYearly' in policy && typeof policy.keepYearly !== 'number') return callback(new BoxError(BoxError.BAD_FIELD, 'keepYearly must be a number', { field: 'retentionPolicy' })); - - api(backupConfig.provider).testConfig(backupConfig, callback); -} - -// this skips password check since that policy is only at creation time -function testProviderConfig(backupConfig, callback) { - assert.strictEqual(typeof backupConfig, 'object'); - assert.strictEqual(typeof callback, 'function'); - - var func = api(backupConfig.provider); - if (!func) return callback(new BoxError(BoxError.BAD_FIELD, 'unknown storage provider', { field: 'provider' })); - - api(backupConfig.provider).testConfig(backupConfig, callback); + return storage.api(backupConfig.provider).removePrivateFields(backupConfig); } function generateEncryptionKeysSync(password) { @@ -193,1067 +103,75 @@ function generateEncryptionKeysSync(password) { }; } -function getByIdentifierAndStatePaged(identifier, state, page, perPage, callback) { +async function add(id, data) { + assert(data && typeof data === 'object'); + assert.strictEqual(typeof id, 'string'); + assert(data.encryptionVersion === null || typeof data.encryptionVersion === 'number'); + assert.strictEqual(typeof data.packageVersion, 'string'); + assert.strictEqual(typeof data.type, 'string'); + assert.strictEqual(typeof data.identifier, 'string'); + assert.strictEqual(typeof data.state, 'string'); + assert(Array.isArray(data.dependsOn)); + assert.strictEqual(typeof data.manifest, 'object'); + assert.strictEqual(typeof data.format, 'string'); + + const creationTime = data.creationTime || new Date(); // allow tests to set the time + const manifestJson = JSON.stringify(data.manifest); + + const [error] = await safe(database.query('INSERT INTO backups (id, identifier, encryptionVersion, packageVersion, type, creationTime, state, dependsOn, manifestJson, format) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', + [ id, data.identifier, data.encryptionVersion, data.packageVersion, data.type, creationTime, data.state, data.dependsOn.join(','), manifestJson, data.format ])); + + if (error && error.code === 'ER_DUP_ENTRY') throw new BoxError(BoxError.ALREADY_EXISTS, 'Backup already exists'); + if (error) throw error; +} + +async function getByIdentifierAndStatePaged(identifier, state, page, perPage) { assert.strictEqual(typeof identifier, 'string'); assert.strictEqual(typeof state, 'string'); assert(typeof page === 'number' && page > 0); assert(typeof perPage === 'number' && perPage > 0); - assert.strictEqual(typeof callback, 'function'); - backupdb.getByIdentifierAndStatePaged(identifier, state, page, perPage, function (error, results) { - if (error) return callback(error); + const results = await database.query(`SELECT ${BACKUPS_FIELDS} FROM backups WHERE identifier = ? AND state = ? ORDER BY creationTime DESC LIMIT ?,?`, [ identifier, state, (page-1)*perPage, perPage ]); - callback(null, results); - }); + results.forEach(function (result) { postProcess(result); }); + + return results; } -function get(backupId, callback) { - assert.strictEqual(typeof backupId, 'string'); - assert.strictEqual(typeof callback, 'function'); - - backupdb.get(backupId, function (error, result) { - if (error) return callback(error); - - callback(null, result); - }); -} - -// This is not part of the storage api, since we don't want to pull the "format" logistics into that -function getBackupFilePath(backupConfig, backupId, format) { - assert.strictEqual(typeof backupConfig, 'object'); - assert.strictEqual(typeof backupId, 'string'); - assert.strictEqual(typeof format, 'string'); - - const backupPath = api(backupConfig.provider).getBackupPath(backupConfig); - - if (format === 'tgz') { - const fileType = backupConfig.encryption ? '.tar.gz.enc' : '.tar.gz'; - return path.join(backupPath, backupId+fileType); - } else { - return path.join(backupPath, backupId); - } -} - -function encryptFilePath(filePath, encryption) { - assert.strictEqual(typeof filePath, 'string'); - assert.strictEqual(typeof encryption, 'object'); - - var encryptedParts = filePath.split('/').map(function (part) { - let hmac = crypto.createHmac('sha256', Buffer.from(encryption.filenameHmacKey, 'hex')); - const iv = hmac.update(part).digest().slice(0, 16); // iv has to be deterministic, for our sync (copy) logic to work - const cipher = crypto.createCipheriv('aes-256-cbc', Buffer.from(encryption.filenameKey, 'hex'), iv); - let crypt = cipher.update(part); - crypt = Buffer.concat([ iv, crypt, cipher.final() ]); - - return crypt.toString('base64') // ensures path is valid - .replace(/\//g, '-') // replace '/' of base64 since it conflicts with path separator - .replace(/=/g,''); // strip trailing = padding. this is only needed if we concat base64 strings, which we don't - }); - - return encryptedParts.join('/'); -} - -function decryptFilePath(filePath, encryption) { - assert.strictEqual(typeof filePath, 'string'); - assert.strictEqual(typeof encryption, 'object'); - - let decryptedParts = []; - for (let part of filePath.split('/')) { - part = part + Array(part.length % 4).join('='); // add back = padding - part = part.replace(/-/g, '/'); // replace with '/' - - try { - const buffer = Buffer.from(part, 'base64'); - const iv = buffer.slice(0, 16); - let decrypt = crypto.createDecipheriv('aes-256-cbc', Buffer.from(encryption.filenameKey, 'hex'), iv); - const plainText = decrypt.update(buffer.slice(16)); - const plainTextString = Buffer.concat([ plainText, decrypt.final() ]).toString('utf8'); - const hmac = crypto.createHmac('sha256', Buffer.from(encryption.filenameHmacKey, 'hex')); - if (!hmac.update(plainTextString).digest().slice(0, 16).equals(iv)) return { error: new BoxError(BoxError.CRYPTO_ERROR, `mac error decrypting part ${part} of path ${filePath}`) }; - - decryptedParts.push(plainTextString); - } catch (error) { - debug(`Error decrypting part ${part} of path ${filePath}:`, error); - return { error: new BoxError(BoxError.CRYPTO_ERROR, `Error decrypting part ${part} of path ${filePath}: ${error.message}`) }; - } - } - - return { result: decryptedParts.join('/') }; -} - -class EncryptStream extends TransformStream { - constructor(encryption) { - super(); - this._headerPushed = false; - this._iv = crypto.randomBytes(16); - this._cipher = crypto.createCipheriv('aes-256-cbc', Buffer.from(encryption.dataKey, 'hex'), this._iv); - this._hmac = crypto.createHmac('sha256', Buffer.from(encryption.dataHmacKey, 'hex')); - } - - pushHeaderIfNeeded() { - if (!this._headerPushed) { - const magic = Buffer.from('CBV2'); - this.push(magic); - this._hmac.update(magic); - this.push(this._iv); - this._hmac.update(this._iv); - this._headerPushed = true; - } - } - - _transform(chunk, ignoredEncoding, callback) { - this.pushHeaderIfNeeded(); - - try { - const crypt = this._cipher.update(chunk); - this._hmac.update(crypt); - callback(null, crypt); - } catch (error) { - callback(error); - } - } - - _flush(callback) { - try { - this.pushHeaderIfNeeded(); // for 0-length files - const crypt = this._cipher.final(); - this.push(crypt); - this._hmac.update(crypt); - callback(null, this._hmac.digest()); // +32 bytes - } catch (error) { - callback(error); - } - } -} - -class DecryptStream extends TransformStream { - constructor(encryption) { - super(); - this._key = Buffer.from(encryption.dataKey, 'hex'); - this._header = Buffer.alloc(0); - this._decipher = null; - this._hmac = crypto.createHmac('sha256', Buffer.from(encryption.dataHmacKey, 'hex')); - this._buffer = Buffer.alloc(0); - } - - _transform(chunk, ignoredEncoding, callback) { - const needed = 20 - this._header.length; // 4 for magic, 16 for iv - - if (this._header.length !== 20) { // not gotten header yet - this._header = Buffer.concat([this._header, chunk.slice(0, needed)]); - if (this._header.length !== 20) return callback(); - - if (!this._header.slice(0, 4).equals(new Buffer.from('CBV2'))) return callback(new BoxError(BoxError.CRYPTO_ERROR, 'Invalid magic in header')); - - const iv = this._header.slice(4); - this._decipher = crypto.createDecipheriv('aes-256-cbc', this._key, iv); - this._hmac.update(this._header); - } - - this._buffer = Buffer.concat([ this._buffer, chunk.slice(needed) ]); - if (this._buffer.length < 32) return callback(); // hmac trailer length is 32 - - try { - const cipherText = this._buffer.slice(0, -32); - this._hmac.update(cipherText); - const plainText = this._decipher.update(cipherText); - this._buffer = this._buffer.slice(-32); - callback(null, plainText); - } catch (error) { - callback(error); - } - } - - _flush (callback) { - if (this._buffer.length !== 32) return callback(new BoxError(BoxError.CRYPTO_ERROR, 'Invalid password or tampered file (not enough data)')); - - try { - if (!this._hmac.digest().equals(this._buffer)) return callback(new BoxError(BoxError.CRYPTO_ERROR, 'Invalid password or tampered file (mac mismatch)')); - - const plainText = this._decipher.final(); - callback(null, plainText); - } catch (error) { - callback(error); - } - } -} - -function createReadStream(sourceFile, encryption) { - assert.strictEqual(typeof sourceFile, 'string'); - assert.strictEqual(typeof encryption, 'object'); - - var stream = fs.createReadStream(sourceFile); - var ps = progressStream({ time: 10000 }); // display a progress every 10 seconds - - stream.on('error', function (error) { - debug(`createReadStream: read stream error at ${sourceFile}`, error); - ps.emit('error', new BoxError(BoxError.FS_ERROR, `Error reading ${sourceFile}: ${error.message} ${error.code}`)); - }); - - stream.on('open', () => ps.emit('open')); - - if (encryption) { - let encryptStream = new EncryptStream(encryption); - - encryptStream.on('error', function (error) { - debug(`createReadStream: encrypt stream error ${sourceFile}`, error); - ps.emit('error', new BoxError(BoxError.CRYPTO_ERROR, `Encryption error at ${sourceFile}: ${error.message}`)); - }); - - return stream.pipe(encryptStream).pipe(ps); - } else { - return stream.pipe(ps); - } -} - -function createWriteStream(destFile, encryption) { - assert.strictEqual(typeof destFile, 'string'); - assert.strictEqual(typeof encryption, 'object'); - - var stream = fs.createWriteStream(destFile); - var ps = progressStream({ time: 10000 }); // display a progress every 10 seconds - - stream.on('error', function (error) { - debug(`createWriteStream: write stream error ${destFile}`, error); - ps.emit('error', new BoxError(BoxError.FS_ERROR, `Write error ${destFile}: ${error.message}`)); - }); - - stream.on('finish', function () { - debug('createWriteStream: done.'); - // we use a separate event because ps is a through2 stream which emits 'finish' event indicating end of inStream and not write - ps.emit('done'); - }); - - if (encryption) { - let decrypt = new DecryptStream(encryption); - decrypt.on('error', function (error) { - debug(`createWriteStream: decrypt stream error ${destFile}`, error); - ps.emit('error', new BoxError(BoxError.CRYPTO_ERROR, `Decryption error at ${destFile}: ${error.message}`)); - }); - - ps.pipe(decrypt).pipe(stream); - } else { - ps.pipe(stream); - } - - return ps; -} - -function tarPack(dataLayout, encryption, callback) { - assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout'); - assert.strictEqual(typeof encryption, 'object'); - assert.strictEqual(typeof callback, 'function'); - - var pack = tar.pack('/', { - dereference: false, // pack the symlink and not what it points to - entries: dataLayout.localPaths(), - ignoreStatError: (path, err) => { - debug(`tarPack: error stat'ing ${path} - ${err.code}`); - return err.code === 'ENOENT'; // ignore if file or dir got removed (probably some temporary file) - }, - map: function(header) { - header.name = dataLayout.toRemotePath(header.name); - // the tar pax format allows us to encode filenames > 100 and size > 8GB (see #640) - // https://www.systutorials.com/docs/linux/man/5-star/ - if (header.size > 8589934590 || header.name > 99) header.pax = { size: header.size }; - return header; - }, - strict: false // do not error for unknown types (skip fifo, char/block devices) - }); - - var gzip = zlib.createGzip({}); - var ps = progressStream({ time: 10000 }); // emit 'progress' every 10 seconds - - pack.on('error', function (error) { - debug('tarPack: tar stream error.', error); - ps.emit('error', new BoxError(BoxError.EXTERNAL_ERROR, error.message)); - }); - - gzip.on('error', function (error) { - debug('tarPack: gzip stream error.', error); - ps.emit('error', new BoxError(BoxError.EXTERNAL_ERROR, error.message)); - }); - - if (encryption) { - const encryptStream = new EncryptStream(encryption); - encryptStream.on('error', function (error) { - debug('tarPack: encrypt stream error.', error); - ps.emit('error', new BoxError(BoxError.EXTERNAL_ERROR, error.message)); - }); - - pack.pipe(gzip).pipe(encryptStream).pipe(ps); - } else { - pack.pipe(gzip).pipe(ps); - } - - return callback(null, ps); -} - -function sync(backupConfig, backupId, dataLayout, progressCallback, callback) { - assert.strictEqual(typeof backupConfig, 'object'); - assert.strictEqual(typeof backupId, 'string'); - assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout'); - assert.strictEqual(typeof progressCallback, 'function'); - assert.strictEqual(typeof callback, 'function'); - - // the number here has to take into account the s3.upload partSize (which is 10MB). So 20=200MB - const concurrency = backupConfig.syncConcurrency || (backupConfig.provider === 's3' ? 20 : 10); - - syncer.sync(dataLayout, function processTask(task, iteratorCallback) { - debug('sync: processing task: %j', task); - // the empty task.path is special to signify the directory - const destPath = task.path && backupConfig.encryption ? encryptFilePath(task.path, backupConfig.encryption) : task.path; - const backupFilePath = path.join(getBackupFilePath(backupConfig, backupId, backupConfig.format), destPath); - - if (task.operation === 'removedir') { - debug(`Removing directory ${backupFilePath}`); - return api(backupConfig.provider).removeDir(backupConfig, backupFilePath) - .on('progress', (message) => progressCallback({ message })) - .on('done', iteratorCallback); - } else if (task.operation === 'remove') { - debug(`Removing ${backupFilePath}`); - return api(backupConfig.provider).remove(backupConfig, backupFilePath, iteratorCallback); - } - - var retryCount = 0; - async.retry({ times: 5, interval: 20000 }, function (retryCallback) { - retryCallback = once(retryCallback); // protect again upload() erroring much later after read stream error - - ++retryCount; - if (task.operation === 'add') { - progressCallback({ message: `Adding ${task.path}` + (retryCount > 1 ? ` (Try ${retryCount})` : '') }); - debug(`Adding ${task.path} position ${task.position} try ${retryCount}`); - var stream = createReadStream(dataLayout.toLocalPath('./' + task.path), backupConfig.encryption); - stream.on('error', (error) => retryCallback(error.message.includes('ENOENT') ? null : error)); // ignore error if file disappears - stream.on('progress', function (progress) { - const transferred = Math.round(progress.transferred/1024/1024), speed = Math.round(progress.speed/1024/1024); - if (!transferred && !speed) return progressCallback({ message: `Uploading ${task.path}` }); // 0M@0MBps looks wrong - progressCallback({ message: `Uploading ${task.path}: ${transferred}M@${speed}MBps` }); // 0M@0MBps looks wrong - }); - // only create the destination path when we have confirmation that the source is available. otherwise, we end up with - // files owned as 'root' and the cp later will fail - stream.on('open', function () { - api(backupConfig.provider).upload(backupConfig, backupFilePath, stream, function (error) { - debug(error ? `Error uploading ${task.path} try ${retryCount}: ${error.message}` : `Uploaded ${task.path}`); - retryCallback(error); - }); - }); - } - }, iteratorCallback); - }, concurrency, function (error) { - if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, error.message)); - - callback(); - }); -} - -// this is not part of 'snapshotting' because we need root access to traverse -function saveFsMetadata(dataLayout, metadataFile, callback) { - assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout'); - assert.strictEqual(typeof metadataFile, 'string'); - assert.strictEqual(typeof callback, 'function'); - - // contains paths prefixed with './' - let metadata = { - emptyDirs: [], - execFiles: [], - symlinks: [] - }; - - // we assume small number of files. spawnSync will raise a ENOBUFS error after maxBuffer - for (let lp of dataLayout.localPaths()) { - const emptyDirs = safe.child_process.execSync(`find ${lp} -type d -empty`, { encoding: 'utf8', maxBuffer: 1024 * 1024 * 30 }); - if (emptyDirs === null) return callback(new BoxError(BoxError.FS_ERROR, `Error finding empty dirs: ${safe.error.message}`)); - if (emptyDirs.length) metadata.emptyDirs = metadata.emptyDirs.concat(emptyDirs.trim().split('\n').map((ed) => dataLayout.toRemotePath(ed))); - - const execFiles = safe.child_process.execSync(`find ${lp} -type f -executable`, { encoding: 'utf8', maxBuffer: 1024 * 1024 * 30 }); - if (execFiles === null) return callback(new BoxError(BoxError.FS_ERROR, `Error finding executables: ${safe.error.message}`)); - if (execFiles.length) metadata.execFiles = metadata.execFiles.concat(execFiles.trim().split('\n').map((ef) => dataLayout.toRemotePath(ef))); - - const symlinks = safe.child_process.execSync(`find ${lp} -type l`, { encoding: 'utf8', maxBuffer: 1024 * 1024 * 30 }); - if (symlinks === null) return callback(new BoxError(BoxError.FS_ERROR, `Error finding symlinks: ${safe.error.message}`)); - if (symlinks.length) metadata.symlinks = metadata.symlinks.concat(symlinks.trim().split('\n').map((sl) => { - const target = safe.fs.readlinkSync(sl); - return { path: dataLayout.toRemotePath(sl), target }; - })); - } - - if (!safe.fs.writeFileSync(metadataFile, JSON.stringify(metadata, null, 4))) return callback(new BoxError(BoxError.FS_ERROR, `Error writing fs metadata: ${safe.error.message}`)); - - callback(); -} - -// this function is called via backupupload (since it needs root to traverse app's directory) -function upload(backupId, format, dataLayoutString, progressCallback, callback) { - assert.strictEqual(typeof backupId, 'string'); - assert.strictEqual(typeof format, 'string'); - assert.strictEqual(typeof dataLayoutString, 'string'); - assert.strictEqual(typeof progressCallback, 'function'); - assert.strictEqual(typeof callback, 'function'); - - debug(`upload: id ${backupId} format ${format} dataLayout ${dataLayoutString}`); - - const dataLayout = DataLayout.fromString(dataLayoutString); - - settings.getBackupConfig(function (error, backupConfig) { - if (error) return callback(error); - - api(backupConfig.provider).checkPreconditions(backupConfig, dataLayout, function (error) { - if (error) return callback(error); - - if (format === 'tgz') { - async.retry({ times: 5, interval: 20000 }, function (retryCallback) { - retryCallback = once(retryCallback); // protect again upload() erroring much later after tar stream error - - tarPack(dataLayout, backupConfig.encryption, function (error, tarStream) { - if (error) return retryCallback(error); - - tarStream.on('progress', function (progress) { - const transferred = Math.round(progress.transferred/1024/1024), speed = Math.round(progress.speed/1024/1024); - if (!transferred && !speed) return progressCallback({ message: 'Uploading backup' }); // 0M@0MBps looks wrong - progressCallback({ message: `Uploading backup ${transferred}M@${speed}MBps` }); - }); - tarStream.on('error', retryCallback); // already returns BoxError - - api(backupConfig.provider).upload(backupConfig, getBackupFilePath(backupConfig, backupId, format), tarStream, retryCallback); - }); - }, callback); - } else { - async.series([ - saveFsMetadata.bind(null, dataLayout, `${dataLayout.localRoot()}/fsmetadata.json`), - sync.bind(null, backupConfig, backupId, dataLayout, progressCallback) - ], callback); - } - }); - }); -} - -function tarExtract(inStream, dataLayout, encryption, callback) { - assert.strictEqual(typeof inStream, 'object'); - assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout'); - assert.strictEqual(typeof encryption, 'object'); - assert.strictEqual(typeof callback, 'function'); - - var gunzip = zlib.createGunzip({}); - var ps = progressStream({ time: 10000 }); // display a progress every 10 seconds - var extract = tar.extract('/', { - map: function (header) { - header.name = dataLayout.toLocalPath(header.name); - return header; - }, - dmode: 500 // ensure directory is writable - }); - - const emitError = once((error) => { - inStream.destroy(); - ps.emit('error', error); - }); - - inStream.on('error', function (error) { - debug('tarExtract: input stream error.', error); - emitError(new BoxError(BoxError.EXTERNAL_ERROR, error.message)); - }); - - gunzip.on('error', function (error) { - debug('tarExtract: gunzip stream error.', error); - emitError(new BoxError(BoxError.EXTERNAL_ERROR, error.message)); - }); - - extract.on('error', function (error) { - debug('tarExtract: extract stream error.', error); - emitError(new BoxError(BoxError.EXTERNAL_ERROR, error.message)); - }); - - extract.on('finish', function () { - debug('tarExtract: done.'); - // we use a separate event because ps is a through2 stream which emits 'finish' event indicating end of inStream and not extract - ps.emit('done'); - }); - - if (encryption) { - let decrypt = new DecryptStream(encryption); - decrypt.on('error', function (error) { - debug('tarExtract: decrypt stream error.', error); - emitError(new BoxError(BoxError.EXTERNAL_ERROR, `Failed to decrypt: ${error.message}`)); - }); - inStream.pipe(ps).pipe(decrypt).pipe(gunzip).pipe(extract); - } else { - inStream.pipe(ps).pipe(gunzip).pipe(extract); - } - - callback(null, ps); -} - -function restoreFsMetadata(dataLayout, metadataFile, callback) { - assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout'); - assert.strictEqual(typeof metadataFile, 'string'); - assert.strictEqual(typeof callback, 'function'); - - debug(`Recreating empty directories in ${dataLayout.toString()}`); - - var metadataJson = safe.fs.readFileSync(metadataFile, 'utf8'); - if (metadataJson === null) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Error loading fsmetadata.json:' + safe.error.message)); - var metadata = safe.JSON.parse(metadataJson); - if (metadata === null) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Error parsing fsmetadata.json:' + safe.error.message)); - - async.eachSeries(metadata.emptyDirs, function createPath(emptyDir, iteratorDone) { - fs.mkdir(dataLayout.toLocalPath(emptyDir), { recursive: true }, iteratorDone); - }, function (error) { - if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `unable to create path: ${error.message}`)); - - async.eachSeries(metadata.execFiles, function createPath(execFile, iteratorDone) { - fs.chmod(dataLayout.toLocalPath(execFile), parseInt('0755', 8), iteratorDone); - }, function (error) { - if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `unable to chmod: ${error.message}`)); - - async.eachSeries(metadata.symlinks || [], function createSymlink(symlink, iteratorDone) { - if (!symlink.target) return iteratorDone(); - // the path may not exist if we had a directory full of symlinks - fs.mkdir(path.dirname(dataLayout.toLocalPath(symlink.path)), { recursive: true }, function (error) { - if (error) return iteratorDone(error); - - fs.symlink(symlink.target, dataLayout.toLocalPath(symlink.path), 'file', iteratorDone); - }); - }, function (error) { - if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `unable to symlink: ${error.message}`)); - - callback(); - }); - }); - }); -} - -function downloadDir(backupConfig, backupFilePath, dataLayout, progressCallback, callback) { - assert.strictEqual(typeof backupConfig, 'object'); - assert.strictEqual(typeof backupFilePath, 'string'); - assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout'); - assert.strictEqual(typeof progressCallback, 'function'); - assert.strictEqual(typeof callback, 'function'); - - debug(`downloadDir: ${backupFilePath} to ${dataLayout.toString()}`); - - function downloadFile(entry, done) { - let relativePath = path.relative(backupFilePath, entry.fullPath); - if (backupConfig.encryption) { - const { error, result } = decryptFilePath(relativePath, backupConfig.encryption); - if (error) return done(new BoxError(BoxError.CRYPTO_ERROR, 'Unable to decrypt file')); - relativePath = result; - } - const destFilePath = dataLayout.toLocalPath('./' + relativePath); - - fs.mkdir(path.dirname(destFilePath), { recursive: true }, function (error) { - if (error) return done(new BoxError(BoxError.FS_ERROR, error.message)); - - async.retry({ times: 5, interval: 20000 }, function (retryCallback) { - api(backupConfig.provider).download(backupConfig, entry.fullPath, function (error, sourceStream) { - if (error) { - progressCallback({ message: `Download ${entry.fullPath} to ${destFilePath} errored: ${error.message}` }); - return retryCallback(error); - } - - let destStream = createWriteStream(destFilePath, backupConfig.encryption); - - // protect against multiple errors. must destroy the write stream so that a previous retry does not write - let closeAndRetry = once((error) => { - if (error) progressCallback({ message: `Download ${entry.fullPath} to ${destFilePath} errored: ${error.message}` }); - else progressCallback({ message: `Download ${entry.fullPath} to ${destFilePath} finished` }); - sourceStream.destroy(); - destStream.destroy(); - retryCallback(error); - }); - - destStream.on('progress', function (progress) { - const transferred = Math.round(progress.transferred/1024/1024), speed = Math.round(progress.speed/1024/1024); - if (!transferred && !speed) return progressCallback({ message: `Downloading ${entry.fullPath}` }); // 0M@0MBps looks wrong - progressCallback({ message: `Downloading ${entry.fullPath}: ${transferred}M@${speed}MBps` }); - }); - destStream.on('error', closeAndRetry); - - sourceStream.on('error', closeAndRetry); - - progressCallback({ message: `Downloading ${entry.fullPath} to ${destFilePath}` }); - - sourceStream.pipe(destStream, { end: true }).on('done', closeAndRetry); - }); - }, done); - }); - } - - api(backupConfig.provider).listDir(backupConfig, backupFilePath, 1000, function (entries, iteratorDone) { - // https://www.digitalocean.com/community/questions/rate-limiting-on-spaces?answer=40441 - const concurrency = backupConfig.downloadConcurrency || (backupConfig.provider === 's3' ? 30 : 10); - - async.eachLimit(entries, concurrency, downloadFile, iteratorDone); - }, callback); -} - -function download(backupConfig, backupId, format, dataLayout, progressCallback, callback) { - assert.strictEqual(typeof backupConfig, 'object'); - assert.strictEqual(typeof backupId, 'string'); - assert.strictEqual(typeof format, 'string'); - assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout'); - assert.strictEqual(typeof progressCallback, 'function'); - assert.strictEqual(typeof callback, 'function'); - - debug(`download: Downloading ${backupId} of format ${format} to ${dataLayout.toString()}`); - - const backupFilePath = getBackupFilePath(backupConfig, backupId, format); - - if (format === 'tgz') { - async.retry({ times: 5, interval: 20000 }, function (retryCallback) { - api(backupConfig.provider).download(backupConfig, backupFilePath, function (error, sourceStream) { - if (error) return retryCallback(error); - - tarExtract(sourceStream, dataLayout, backupConfig.encryption, function (error, ps) { - if (error) return retryCallback(error); - - ps.on('progress', function (progress) { - const transferred = Math.round(progress.transferred/1024/1024), speed = Math.round(progress.speed/1024/1024); - if (!transferred && !speed) return progressCallback({ message: 'Downloading backup' }); // 0M@0MBps looks wrong - progressCallback({ message: `Downloading ${transferred}M@${speed}MBps` }); - }); - ps.on('error', retryCallback); - ps.on('done', retryCallback); - }); - }); - }, callback); - } else { - downloadDir(backupConfig, backupFilePath, dataLayout, progressCallback, function (error) { - if (error) return callback(error); - - restoreFsMetadata(dataLayout, `${dataLayout.localRoot()}/fsmetadata.json`, callback); - }); - } -} - -function restore(backupConfig, backupId, progressCallback, callback) { - assert.strictEqual(typeof backupConfig, 'object'); - assert.strictEqual(typeof backupId, 'string'); - assert.strictEqual(typeof progressCallback, 'function'); - assert.strictEqual(typeof callback, 'function'); - - const boxDataDir = safe.fs.realpathSync(paths.BOX_DATA_DIR); - if (!boxDataDir) return callback(new BoxError(BoxError.FS_ERROR, `Error resolving boxdata: ${safe.error.message}`)); - const dataLayout = new DataLayout(boxDataDir, []); - - download(backupConfig, backupId, backupConfig.format, dataLayout, progressCallback, function (error) { - if (error) return callback(error); - - debug('restore: download completed, importing database'); - - database.importFromFile(`${dataLayout.localRoot()}/box.mysqldump`, function (error) { - if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error)); - - debug('restore: database imported'); - - settings.initCache(callback); - }); - }); -} - -function downloadApp(app, restoreConfig, progressCallback, callback) { - assert.strictEqual(typeof app, 'object'); - assert.strictEqual(typeof restoreConfig, 'object'); - assert.strictEqual(typeof progressCallback, 'function'); - assert.strictEqual(typeof callback, 'function'); - - const appDataDir = safe.fs.realpathSync(path.join(paths.APPS_DATA_DIR, app.id)); - if (!appDataDir) return callback(new BoxError(BoxError.FS_ERROR, safe.error.message)); - const dataLayout = new DataLayout(appDataDir, app.dataDir ? [{ localDir: app.dataDir, remoteDir: 'data' }] : []); - - const startTime = new Date(); - const getBackupConfigFunc = restoreConfig.backupConfig ? (next) => next(null, restoreConfig.backupConfig) : settings.getBackupConfig; - - getBackupConfigFunc(function (error, backupConfig) { - if (error) return callback(error); - - download(backupConfig, restoreConfig.backupId, restoreConfig.backupFormat, dataLayout, progressCallback, function (error) { - debug('downloadApp: time: %s', (new Date() - startTime)/1000); - - callback(error); - }); - }); -} - -function runBackupUpload(uploadConfig, progressCallback, callback) { - assert.strictEqual(typeof uploadConfig, 'object'); - assert.strictEqual(typeof progressCallback, 'function'); - assert.strictEqual(typeof callback, 'function'); - - const { backupId, backupConfig, dataLayout, progressTag } = uploadConfig; - assert.strictEqual(typeof backupId, 'string'); - assert.strictEqual(typeof backupConfig, 'object'); - assert.strictEqual(typeof progressTag, 'string'); - assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout'); - - let result = ''; // the script communicates error result as a string - - // https://stackoverflow.com/questions/48387040/node-js-recommended-max-old-space-size - const envCopy = Object.assign({}, process.env); - if (backupConfig.memoryLimit && backupConfig.memoryLimit >= 2*1024*1024*1024) { - const heapSize = Math.min((backupConfig.memoryLimit/1024/1024) - 256, 8192); - debug(`runBackupUpload: adjusting heap size to ${heapSize}M`); - envCopy.NODE_OPTIONS = `--max-old-space-size=${heapSize}`; - } - - shell.sudo(`backup-${backupId}`, [ BACKUP_UPLOAD_CMD, backupId, backupConfig.format, dataLayout.toString() ], { env: envCopy, preserveEnv: true, ipc: true }, function (error) { - if (error && (error.code === null /* signal */ || (error.code !== 0 && error.code !== 50))) { // backuptask crashed - return callback(new BoxError(BoxError.INTERNAL_ERROR, 'Backuptask crashed')); - } else if (error && error.code === 50) { // exited with error - return callback(new BoxError(BoxError.EXTERNAL_ERROR, result)); - } - - callback(); - }).on('message', function (progress) { // this is { message } or { result } - if ('message' in progress) return progressCallback({ message: `${progress.message} (${progressTag})` }); - debug(`runBackupUpload: result - ${JSON.stringify(progress)}`); - result = progress.result; - }); -} - -function getSnapshotInfo(id) { +async function get(id) { assert.strictEqual(typeof id, 'string'); - var contents = safe.fs.readFileSync(paths.SNAPSHOT_INFO_FILE, 'utf8'); - var info = safe.JSON.parse(contents); - if (!info) return { }; - return info[id] || { }; + const result = await database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups WHERE id = ? ORDER BY creationTime DESC', [ id ]); + if (result.length === 0) return null; + + return postProcess(result[0]); } -function setSnapshotInfo(id, info, callback) { +async function getByTypePaged(type, page, perPage) { + assert.strictEqual(typeof type, 'string'); + assert(typeof page === 'number' && page > 0); + assert(typeof perPage === 'number' && perPage > 0); + + const results = await database.query(`SELECT ${BACKUPS_FIELDS} FROM backups WHERE type = ? ORDER BY creationTime DESC LIMIT ?,?`, [ type, (page-1)*perPage, perPage ]); + + results.forEach(function (result) { postProcess(result); }); + + return results; +} + +async function update(id, backup) { assert.strictEqual(typeof id, 'string'); - assert.strictEqual(typeof info, 'object'); - assert.strictEqual(typeof callback, 'function'); + assert.strictEqual(typeof backup, 'object'); - var contents = safe.fs.readFileSync(paths.SNAPSHOT_INFO_FILE, 'utf8'); - var data = safe.JSON.parse(contents) || { }; - if (info) data[id] = info; else delete data[id]; - if (!safe.fs.writeFileSync(paths.SNAPSHOT_INFO_FILE, JSON.stringify(data, null, 4), 'utf8')) { - return callback(new BoxError(BoxError.FS_ERROR, safe.error.message)); + let fields = [ ], values = [ ]; + for (const p in backup) { + fields.push(p + ' = ?'); + values.push(backup[p]); } + values.push(id); - callback(); -} - -function snapshotBox(progressCallback, callback) { - assert.strictEqual(typeof progressCallback, 'function'); - assert.strictEqual(typeof callback, 'function'); - - progressCallback({ message: 'Snapshotting box' }); - - const startTime = new Date(); - - database.exportToFile(`${paths.BOX_DATA_DIR}/box.mysqldump`, function (error) { - if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error)); - - debug(`snapshotBox: took ${(new Date() - startTime)/1000} seconds`); - - return callback(); - }); -} - -function uploadBoxSnapshot(backupConfig, progressCallback, callback) { - assert.strictEqual(typeof backupConfig, 'object'); - assert.strictEqual(typeof progressCallback, 'function'); - assert.strictEqual(typeof callback, 'function'); - - snapshotBox(progressCallback, function (error) { - if (error) return callback(error); - - const boxDataDir = safe.fs.realpathSync(paths.BOX_DATA_DIR); - if (!boxDataDir) return callback(new BoxError(BoxError.FS_ERROR, `Error resolving boxdata: ${safe.error.message}`)); - - const uploadConfig = { - backupId: 'snapshot/box', - backupConfig, - dataLayout: new DataLayout(boxDataDir, []), - progressTag: 'box' - }; - - progressCallback({ message: 'Uploading box snapshot' }); - - const startTime = new Date(); - - runBackupUpload(uploadConfig, progressCallback, function (error) { - if (error) return callback(error); - - debug(`uploadBoxSnapshot: took ${(new Date() - startTime)/1000} seconds`); - - setSnapshotInfo('box', { timestamp: new Date().toISOString(), format: backupConfig.format }, callback); - }); - }); -} - -function rotateBoxBackup(backupConfig, tag, options, appBackupIds, progressCallback, callback) { - assert.strictEqual(typeof backupConfig, 'object'); - assert.strictEqual(typeof tag, 'string'); - assert.strictEqual(typeof options, 'object'); - assert(Array.isArray(appBackupIds)); - assert.strictEqual(typeof progressCallback, 'function'); - assert.strictEqual(typeof callback, 'function'); - - const backupId = `${tag}/box_v${constants.VERSION}`; - const format = backupConfig.format; - - debug(`Rotating box backup to id ${backupId}`); - - const data = { - encryptionVersion: backupConfig.encryption ? 2 : null, - packageVersion: constants.VERSION, - type: exports.BACKUP_TYPE_BOX, - state: exports.BACKUP_STATE_CREATING, - identifier: 'box', - dependsOn: appBackupIds, - manifest: null, - format: format - }; - backupdb.add(backupId, data, function (error) { - if (error) return callback(error); - - var copy = api(backupConfig.provider).copy(backupConfig, getBackupFilePath(backupConfig, 'snapshot/box', format), getBackupFilePath(backupConfig, backupId, format)); - copy.on('progress', (message) => progressCallback({ message: `box: ${message}` })); - copy.on('done', function (copyBackupError) { - const state = copyBackupError ? exports.BACKUP_STATE_ERROR : exports.BACKUP_STATE_NORMAL; - - backupdb.update(backupId, { preserveSecs: options.preserveSecs || 0, state }, function (error) { - if (copyBackupError) return callback(copyBackupError); - if (error) return callback(error); - - debug(`Rotated box backup successfully as id ${backupId}`); - - callback(null, backupId); - }); - }); - }); -} - -function backupBoxWithAppBackupIds(appBackupIds, tag, options, progressCallback, callback) { - assert(Array.isArray(appBackupIds)); - assert.strictEqual(typeof tag, 'string'); - assert.strictEqual(typeof options, 'object'); - assert.strictEqual(typeof progressCallback, 'function'); - assert.strictEqual(typeof callback, 'function'); - - settings.getBackupConfig(function (error, backupConfig) { - if (error) return callback(error); - - uploadBoxSnapshot(backupConfig, progressCallback, function (error) { - if (error) return callback(error); - - rotateBoxBackup(backupConfig, tag, options, appBackupIds, progressCallback, callback); - }); - }); -} - -function canBackupApp(app) { - // only backup apps that are installed or specific pending states - - // stopped apps cannot be backed up because addons might be down (redis) - if (app.runState === apps.RSTATE_STOPPED) return false; - - // we used to check the health here but that doesn't work for stopped apps. it's better to just fail - // and inform the user if the backup fails and the app addons have not been setup yet. - return app.installationState === apps.ISTATE_INSTALLED || - app.installationState === apps.ISTATE_PENDING_CONFIGURE || - app.installationState === apps.ISTATE_PENDING_BACKUP || // called from apptask - app.installationState === apps.ISTATE_PENDING_UPDATE; // called from apptask -} - -function snapshotApp(app, progressCallback, callback) { - assert.strictEqual(typeof app, 'object'); - assert.strictEqual(typeof progressCallback, 'function'); - assert.strictEqual(typeof callback, 'function'); - - const startTime = new Date(); - progressCallback({ message: `Snapshotting app ${app.fqdn}` }); - - apps.backupConfig(app, function (error) { - if (error) return callback(error); - - services.backupAddons(app, app.manifest.addons, function (error) { - if (error) return callback(error); - - debugApp(app, `snapshotApp: took ${(new Date() - startTime)/1000} seconds`); - - return callback(null); - }); - }); -} - -function rotateAppBackup(backupConfig, app, tag, options, progressCallback, callback) { - assert.strictEqual(typeof backupConfig, 'object'); - assert.strictEqual(typeof app, 'object'); - assert.strictEqual(typeof tag, 'string'); - assert.strictEqual(typeof options, 'object'); - assert.strictEqual(typeof progressCallback, 'function'); - assert.strictEqual(typeof callback, 'function'); - - const startTime = new Date(); - - const snapshotInfo = getSnapshotInfo(app.id); - - const manifest = snapshotInfo.restoreConfig ? snapshotInfo.restoreConfig.manifest : snapshotInfo.manifest; // compat - const backupId = `${tag}/app_${app.fqdn}_v${manifest.version}`; - const format = backupConfig.format; - - debug(`Rotating app backup of ${app.id} to id ${backupId}`); - - const data = { - encryptionVersion: backupConfig.encryption ? 2 : null, - packageVersion: manifest.version, - type: exports.BACKUP_TYPE_APP, - state: exports.BACKUP_STATE_CREATING, - identifier: app.id, - dependsOn: [ ], - manifest, - format: format - }; - - backupdb.add(backupId, data, function (error) { - if (error) return callback(error); - - const copy = api(backupConfig.provider).copy(backupConfig, getBackupFilePath(backupConfig, `snapshot/app_${app.id}`, format), getBackupFilePath(backupConfig, backupId, format)); - copy.on('progress', (message) => progressCallback({ message: `${message} (${app.fqdn})` })); - copy.on('done', function (copyBackupError) { - const state = copyBackupError ? exports.BACKUP_STATE_ERROR : exports.BACKUP_STATE_NORMAL; - - backupdb.update(backupId, { preserveSecs: options.preserveSecs || 0, state }, function (error) { - if (copyBackupError) return callback(copyBackupError); - if (error) return callback(error); - - debug(`Rotated app backup of ${app.id} successfully to id ${backupId}. Took ${(new Date() - startTime)/1000} seconds`); - - callback(null, backupId); - }); - }); - }); -} - -function uploadAppSnapshot(backupConfig, app, progressCallback, callback) { - assert.strictEqual(typeof backupConfig, 'object'); - assert.strictEqual(typeof app, 'object'); - assert.strictEqual(typeof progressCallback, 'function'); - assert.strictEqual(typeof callback, 'function'); - - snapshotApp(app, progressCallback, function (error) { - if (error) return callback(error); - - const backupId = util.format('snapshot/app_%s', app.id); - const appDataDir = safe.fs.realpathSync(path.join(paths.APPS_DATA_DIR, app.id)); - if (!appDataDir) return callback(new BoxError(BoxError.FS_ERROR, `Error resolving appsdata: ${safe.error.message}`)); - - const dataLayout = new DataLayout(appDataDir, app.dataDir ? [{ localDir: app.dataDir, remoteDir: 'data' }] : []); - - progressCallback({ message: `Uploading app snapshot ${app.fqdn}`}); - - const uploadConfig = { - backupId, - backupConfig, - dataLayout, - progressTag: app.fqdn - }; - - const startTime = new Date(); - - runBackupUpload(uploadConfig, progressCallback, function (error) { - if (error) return callback(error); - - debugApp(app, `uploadAppSnapshot: ${backupId} done. ${(new Date() - startTime)/1000} seconds`); - - setSnapshotInfo(app.id, { timestamp: new Date().toISOString(), manifest: app.manifest, format: backupConfig.format }, callback); - }); - }); -} - -function backupAppWithTag(app, tag, options, progressCallback, callback) { - assert.strictEqual(typeof app, 'object'); - assert.strictEqual(typeof tag, 'string'); - assert.strictEqual(typeof options, 'object'); - assert.strictEqual(typeof progressCallback, 'function'); - assert.strictEqual(typeof callback, 'function'); - - if (!canBackupApp(app)) { // if we cannot backup, reuse it's most recent backup - getByIdentifierAndStatePaged(app.id, exports.BACKUP_STATE_NORMAL, 1, 1, function (error, results) { - if (error) return callback(error); - if (results.length === 0) return callback(null, null); // no backup to re-use - - callback(null, results[0].id); - }); - - return; - } - - settings.getBackupConfig(function (error, backupConfig) { - if (error) return callback(error); - - uploadAppSnapshot(backupConfig, app, progressCallback, function (error) { - if (error) return callback(error); - - rotateAppBackup(backupConfig, app, tag, options, progressCallback, callback); - }); - }); -} - -function backupApp(app, options, progressCallback, callback) { - assert.strictEqual(typeof app, 'object'); - assert.strictEqual(typeof options, 'object'); - assert.strictEqual(typeof progressCallback, 'function'); - assert.strictEqual(typeof callback, 'function'); - - if (options.snapshotOnly) return snapshotApp(app, progressCallback, callback); - - const tag = (new Date()).toISOString().replace(/[T.]/g, '-').replace(/[:Z]/g,''); - - debug(`backupApp - Backing up ${app.fqdn} with tag ${tag}`); - - backupAppWithTag(app, tag, options, progressCallback, callback); -} - -// this function expects you to have a lock. Unlike other progressCallback this also has a progress field -function backupBoxAndApps(options, progressCallback, callback) { - assert.strictEqual(typeof options, 'object'); - assert.strictEqual(typeof progressCallback, 'function'); - assert.strictEqual(typeof callback, 'function'); - - const tag = (new Date()).toISOString().replace(/[T.]/g, '-').replace(/[:Z]/g,''); - - apps.getAll(function (error, allApps) { - if (error) return callback(error); - - let percent = 1; - let step = 100/(allApps.length+2); - - async.mapSeries(allApps, function iterator(app, iteratorCallback) { - progressCallback({ percent: percent, message: `Backing up ${app.fqdn}` }); - percent += step; - - if (!app.enableBackup) { - debug(`Skipped backup ${app.fqdn}`); - return iteratorCallback(null, null); // nothing to backup - } - - const startTime = new Date(); - backupAppWithTag(app, tag, options, (progress) => progressCallback({ percent: percent, message: progress.message }), function (error, backupId) { - if (error) { - debugApp(app, 'Unable to backup', error); - return iteratorCallback(error); - } - - debugApp(app, `Backed up. Took ${(new Date() - startTime)/1000} seconds`); - - iteratorCallback(null, backupId || null); // clear backupId if is in BAD_STATE and never backed up - }); - }, function appsBackedUp(error, backupIds) { - if (error) return callback(error); - - backupIds = backupIds.filter(function (id) { return id !== null; }); // remove apps in bad state that were never backed up - - progressCallback({ percent: percent, message: 'Backing up system data' }); - percent += step; - - backupBoxWithAppBackupIds(backupIds, tag, options, (progress) => progressCallback({ percent: percent, message: progress.message }), callback); - }); - }); + const result = await database.query('UPDATE backups SET ' + fields.join(', ') + ' WHERE id = ?', values); + if (result.affectedRows !== 1) throw new BoxError(BoxError.NOT_FOUND, 'Backup not found'); } function startBackupTask(auditSource, callback) { @@ -1283,214 +201,22 @@ function startBackupTask(auditSource, callback) { }); } -function applyBackupRetentionPolicy(backups, policy, referencedBackupIds) { - assert(Array.isArray(backups)); - assert.strictEqual(typeof policy, 'object'); - assert(Array.isArray(referencedBackupIds)); +async function list(page, perPage) { + assert(typeof page === 'number' && page > 0); + assert(typeof perPage === 'number' && perPage > 0); - const now = new Date(); + const results = await database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups ORDER BY creationTime DESC LIMIT ?,?', [ (page-1)*perPage, perPage ]); - for (const backup of backups) { - if (backup.state === exports.BACKUP_STATE_ERROR) { - backup.discardReason = 'error'; - } else if (backup.state === exports.BACKUP_STATE_CREATING) { - if ((now - backup.creationTime) < 48*60*60*1000) backup.keepReason = 'creating'; - else backup.discardReason = 'creating-too-long'; - } else if (referencedBackupIds.includes(backup.id)) { - backup.keepReason = 'reference'; - } else if ((now - backup.creationTime) < (backup.preserveSecs * 1000)) { - backup.keepReason = 'preserveSecs'; - } else if ((now - backup.creationTime < policy.keepWithinSecs * 1000) || policy.keepWithinSecs < 0) { - backup.keepReason = 'keepWithinSecs'; - } - } + results.forEach(function (result) { postProcess(result); }); - const KEEP_FORMATS = { - keepDaily: 'Y-M-D', - keepWeekly: 'Y-W', - keepMonthly: 'Y-M', - keepYearly: 'Y' - }; - - for (const format of [ 'keepDaily', 'keepWeekly', 'keepMonthly', 'keepYearly' ]) { - if (!(format in policy)) continue; - - const n = policy[format]; // we want to keep "n" backups of format - if (!n) continue; // disabled rule - - let lastPeriod = null, keptSoFar = 0; - for (const backup of backups) { - if (backup.discardReason) continue; // already discarded for some reason - if (backup.keepReason && backup.keepReason !== 'reference') continue; // kept for some other reason - const period = moment(backup.creationTime).format(KEEP_FORMATS[format]); - if (period === lastPeriod) continue; // already kept for this period - - lastPeriod = period; - backup.keepReason = backup.keepReason ? `${backup.keepReason}+${format}` : format; - if (++keptSoFar === n) break; - } - } - - if (policy.keepLatest) { - let latestNormalBackup = backups.find(b => b.state === exports.BACKUP_STATE_NORMAL); - if (latestNormalBackup && !latestNormalBackup.keepReason) latestNormalBackup.keepReason = 'latest'; - } - - for (const backup of backups) { - debug(`applyBackupRetentionPolicy: ${backup.id} ${backup.type} ${backup.keepReason || backup.discardReason || 'unprocessed'}`); - } + return results; } -function cleanupBackup(backupConfig, backup, progressCallback, callback) { - assert.strictEqual(typeof backupConfig, 'object'); - assert.strictEqual(typeof backup, 'object'); - assert.strictEqual(typeof progressCallback, 'function'); - assert.strictEqual(typeof callback, 'function'); +async function del(id) { + assert.strictEqual(typeof id, 'string'); - const backupFilePath = getBackupFilePath(backupConfig, backup.id, backup.format); - - function done(error) { - if (error) { - debug('cleanupBackup: error removing backup %j : %s', backup, error.message); - return callback(); - } - - // prune empty directory if possible - api(backupConfig.provider).remove(backupConfig, path.dirname(backupFilePath), function (error) { - if (error) debug('cleanupBackup: unable to prune backup directory %s : %s', path.dirname(backupFilePath), error.message); - - backupdb.del(backup.id, function (error) { - if (error) debug('cleanupBackup: error removing from database', error); - else debug('cleanupBackup: removed %s', backup.id); - - callback(); - }); - }); - } - - if (backup.format ==='tgz') { - progressCallback({ message: `${backup.id}: Removing ${backupFilePath}`}); - api(backupConfig.provider).remove(backupConfig, backupFilePath, done); - } else { - var events = api(backupConfig.provider).removeDir(backupConfig, backupFilePath); - events.on('progress', (message) => progressCallback({ message: `${backup.id}: ${message}` })); - events.on('done', done); - } -} - -function cleanupAppBackups(backupConfig, referencedAppBackupIds, progressCallback, callback) { - assert.strictEqual(typeof backupConfig, 'object'); - assert(Array.isArray(referencedAppBackupIds)); - assert.strictEqual(typeof progressCallback, 'function'); - assert.strictEqual(typeof callback, 'function'); - - let removedAppBackupIds = []; - - apps.getAll(function (error, allApps) { - if (error) return callback(error); - - const allAppIds = allApps.map(a => a.id); - - backupdb.getByTypePaged(exports.BACKUP_TYPE_APP, 1, 1000, function (error, appBackups) { - if (error) return callback(error); - - // collate the backups by app id. note that the app could already have been uninstalled - let appBackupsById = {}; - for (const appBackup of appBackups) { - if (!appBackupsById[appBackup.identifier]) appBackupsById[appBackup.identifier] = []; - appBackupsById[appBackup.identifier].push(appBackup); - } - - // apply backup policy per app. keep latest backup only for existing apps - let appBackupsToRemove = []; - for (const appId of Object.keys(appBackupsById)) { - applyBackupRetentionPolicy(appBackupsById[appId], _.extend({ keepLatest: allAppIds.includes(appId) }, backupConfig.retentionPolicy), referencedAppBackupIds); - appBackupsToRemove = appBackupsToRemove.concat(appBackupsById[appId].filter(b => !b.keepReason)); - } - - async.eachSeries(appBackupsToRemove, function iterator(appBackup, iteratorDone) { - progressCallback({ message: `Removing app backup (${appBackup.identifier}): ${appBackup.id}`}); - removedAppBackupIds.push(appBackup.id); - cleanupBackup(backupConfig, appBackup, progressCallback, iteratorDone); - }, function () { - debug('cleanupAppBackups: done'); - - callback(null, removedAppBackupIds); - }); - }); - }); -} - -function cleanupBoxBackups(backupConfig, progressCallback, callback) { - assert.strictEqual(typeof backupConfig, 'object'); - assert.strictEqual(typeof progressCallback, 'function'); - assert.strictEqual(typeof callback, 'function'); - - let referencedAppBackupIds = [], removedBoxBackupIds = []; - - backupdb.getByTypePaged(exports.BACKUP_TYPE_BOX, 1, 1000, function (error, boxBackups) { - if (error) return callback(error); - - applyBackupRetentionPolicy(boxBackups, _.extend({ keepLatest: true }, backupConfig.retentionPolicy), [] /* references */); - - async.eachSeries(boxBackups, function iterator(boxBackup, iteratorNext) { - if (boxBackup.keepReason) { - referencedAppBackupIds = referencedAppBackupIds.concat(boxBackup.dependsOn); - return iteratorNext(); - } - - progressCallback({ message: `Removing box backup ${boxBackup.id}`}); - - removedBoxBackupIds.push(boxBackup.id); - cleanupBackup(backupConfig, boxBackup, progressCallback, iteratorNext); - }, function () { - debug('cleanupBoxBackups: done'); - - callback(null, { removedBoxBackupIds, referencedAppBackupIds }); - }); - }); -} - -function cleanupMissingBackups(backupConfig, progressCallback, callback) { - assert.strictEqual(typeof backupConfig, 'object'); - assert.strictEqual(typeof progressCallback, 'function'); - assert.strictEqual(typeof callback, 'function'); - - let page = 1, perPage = 1000, more = false, missingBackupIds = []; - - if (constants.TEST) return callback(null, missingBackupIds); - - async.doWhilst(function (whilstCallback) { - backupdb.list(page, perPage, function (error, result) { - if (error) return whilstCallback(error); - - async.eachSeries(result, function (backup, next) { - let backupFilePath = getBackupFilePath(backupConfig, backup.id, backup.format); - if (backup.format === 'rsync') backupFilePath = backupFilePath + '/'; // add trailing slash to indicate directory - - api(backupConfig.provider).exists(backupConfig, backupFilePath, function (error, exists) { - if (error || exists) return next(); - - progressCallback({ message: `Removing missing backup ${backup.id}`}); - - backupdb.del(backup.id, function (error) { - if (error) debug(`cleanupBackup: error removing ${backup.id} from database`, error); - - missingBackupIds.push(backup.id); - - next(); - }); - }); - }, function () { - more = result.length === perPage; - whilstCallback(); - }); - }); - }, function (testDone) { return testDone(null, more); }, function (error) { - if (error) return callback(error); - - return callback(null, missingBackupIds); - }); + const result = await database.query('DELETE FROM backups WHERE id=?', [ id ]); + if (result.affectedRows !== 1) throw new BoxError(BoxError.NOT_FOUND, 'Backup not found'); } function cleanupCacheFilesSync() { @@ -1502,84 +228,28 @@ function cleanupCacheFilesSync() { }); } -// removes the snapshots of apps that have been uninstalled -function cleanupSnapshots(backupConfig, callback) { - assert.strictEqual(typeof backupConfig, 'object'); - assert.strictEqual(typeof callback, 'function'); +function getSnapshotInfo(id) { + assert.strictEqual(typeof id, 'string'); - var contents = safe.fs.readFileSync(paths.SNAPSHOT_INFO_FILE, 'utf8'); - var info = safe.JSON.parse(contents); - if (!info) return callback(); - - delete info.box; - async.eachSeries(Object.keys(info), function (appId, iteratorDone) { - apps.get(appId, function (error /*, app */) { - if (!error || error.reason !== BoxError.NOT_FOUND) return iteratorDone(); - - function done(/* ignoredError */) { - safe.fs.unlinkSync(path.join(paths.BACKUP_INFO_DIR, `${appId}.sync.cache`)); - safe.fs.unlinkSync(path.join(paths.BACKUP_INFO_DIR, `${appId}.sync.cache.new`)); - - setSnapshotInfo(appId, null, function (/* ignoredError */) { - debug('cleanupSnapshots: cleaned up snapshot of app id %s', appId); - - iteratorDone(); - }); - } - - if (info[appId].format ==='tgz') { - api(backupConfig.provider).remove(backupConfig, getBackupFilePath(backupConfig, `snapshot/app_${appId}`, info[appId].format), done); - } else { - var events = api(backupConfig.provider).removeDir(backupConfig, getBackupFilePath(backupConfig, `snapshot/app_${appId}`, info[appId].format)); - events.on('progress', function (detail) { debug(`cleanupSnapshots: ${detail}`); }); - events.on('done', done); - } - }); - }, function () { - debug('cleanupSnapshots: done'); - - callback(); - }); + const contents = safe.fs.readFileSync(paths.SNAPSHOT_INFO_FILE, 'utf8'); + const info = safe.JSON.parse(contents); + if (!info) return { }; + return info[id] || { }; } -function cleanup(progressCallback, callback) { - assert.strictEqual(typeof progressCallback, 'function'); +function setSnapshotInfo(id, info, callback) { + assert.strictEqual(typeof id, 'string'); + assert.strictEqual(typeof info, 'object'); assert.strictEqual(typeof callback, 'function'); - settings.getBackupConfig(function (error, backupConfig) { - if (error) return callback(error); + const contents = safe.fs.readFileSync(paths.SNAPSHOT_INFO_FILE, 'utf8'); + const data = safe.JSON.parse(contents) || { }; + if (info) data[id] = info; else delete data[id]; + if (!safe.fs.writeFileSync(paths.SNAPSHOT_INFO_FILE, JSON.stringify(data, null, 4), 'utf8')) { + return callback(new BoxError(BoxError.FS_ERROR, safe.error.message)); + } - if (backupConfig.retentionPolicy.keepWithinSecs < 0) { - debug('cleanup: keeping all backups'); - return callback(null, {}); - } - - progressCallback({ percent: 10, message: 'Cleaning box backups' }); - - cleanupBoxBackups(backupConfig, progressCallback, function (error, { removedBoxBackupIds, referencedAppBackupIds }) { - if (error) return callback(error); - - progressCallback({ percent: 40, message: 'Cleaning app backups' }); - - cleanupAppBackups(backupConfig, referencedAppBackupIds, progressCallback, function (error, removedAppBackupIds) { - if (error) return callback(error); - - progressCallback({ percent: 70, message: 'Cleaning missing backups' }); - - cleanupMissingBackups(backupConfig, progressCallback, function (error, missingBackupIds) { - if (error) return callback(error); - - progressCallback({ percent: 90, message: 'Cleaning snapshots' }); - - cleanupSnapshots(backupConfig, function (error) { - if (error) return callback(error); - - callback(null, { removedBoxBackupIds, removedAppBackupIds, missingBackupIds }); - }); - }); - }); - }); - }); + callback(); } async function startCleanupTask(auditSource) { @@ -1611,3 +281,43 @@ function configureCollectd(backupConfig, callback) { collectd.removeProfile('cloudron-backup', callback); } } + +function testConfig(backupConfig, callback) { + assert.strictEqual(typeof backupConfig, 'object'); + assert.strictEqual(typeof callback, 'function'); + + const func = storage.api(backupConfig.provider); + if (!func) return callback(new BoxError(BoxError.BAD_FIELD, 'unknown storage provider', { field: 'provider' })); + + if (backupConfig.format !== 'tgz' && backupConfig.format !== 'rsync') return callback(new BoxError(BoxError.BAD_FIELD, 'unknown format', { field: 'format' })); + + const job = safe.safeCall(function () { return new CronJob(backupConfig.schedulePattern); }); + if (!job) return callback(new BoxError(BoxError.BAD_FIELD, 'Invalid schedule pattern', { field: 'schedulePattern' })); + + if ('password' in backupConfig) { + if (typeof backupConfig.password !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'password must be a string', { field: 'password' })); + if (backupConfig.password.length < 8) return callback(new BoxError(BoxError.BAD_FIELD, 'password must be atleast 8 characters', { field: 'password' })); + } + + const policy = backupConfig.retentionPolicy; + if (!policy) return callback(new BoxError(BoxError.BAD_FIELD, 'retentionPolicy is required', { field: 'retentionPolicy' })); + if (!['keepWithinSecs','keepDaily','keepWeekly','keepMonthly','keepYearly'].find(k => !!policy[k])) return callback(new BoxError(BoxError.BAD_FIELD, 'properties missing', { field: 'retentionPolicy' })); + if ('keepWithinSecs' in policy && typeof policy.keepWithinSecs !== 'number') return callback(new BoxError(BoxError.BAD_FIELD, 'keepWithinSecs must be a number', { field: 'retentionPolicy' })); + if ('keepDaily' in policy && typeof policy.keepDaily !== 'number') return callback(new BoxError(BoxError.BAD_FIELD, 'keepDaily must be a number', { field: 'retentionPolicy' })); + if ('keepWeekly' in policy && typeof policy.keepWeekly !== 'number') return callback(new BoxError(BoxError.BAD_FIELD, 'keepWeekly must be a number', { field: 'retentionPolicy' })); + if ('keepMonthly' in policy && typeof policy.keepMonthly !== 'number') return callback(new BoxError(BoxError.BAD_FIELD, 'keepMonthly must be a number', { field: 'retentionPolicy' })); + if ('keepYearly' in policy && typeof policy.keepYearly !== 'number') return callback(new BoxError(BoxError.BAD_FIELD, 'keepYearly must be a number', { field: 'retentionPolicy' })); + + storage.api(backupConfig.provider).testConfig(backupConfig, callback); +} + +// this skips password check since that policy is only at creation time +function testProviderConfig(backupConfig, callback) { + assert.strictEqual(typeof backupConfig, 'object'); + assert.strictEqual(typeof callback, 'function'); + + const func = storage.api(backupConfig.provider); + if (!func) return callback(new BoxError(BoxError.BAD_FIELD, 'unknown storage provider', { field: 'provider' })); + + storage.api(backupConfig.provider).testConfig(backupConfig, callback); +} diff --git a/src/backuptask.js b/src/backuptask.js new file mode 100644 index 000000000..3093b2749 --- /dev/null +++ b/src/backuptask.js @@ -0,0 +1,1044 @@ +'use strict'; + +exports = module.exports = { + backupBoxAndApps, + + restore, + + backupApp, + downloadApp, + + upload, + + _restoreFsMetadata: restoreFsMetadata, + _saveFsMetadata: saveFsMetadata, +}; + +const apps = require('./apps.js'), + assert = require('assert'), + async = require('async'), + backups = require('./backups.js'), + BoxError = require('./boxerror.js'), + constants = require('./constants.js'), + crypto = require('crypto'), + DataLayout = require('./datalayout.js'), + database = require('./database.js'), + debug = require('debug')('box:backuptask'), + fs = require('fs'), + once = require('once'), + path = require('path'), + paths = require('./paths.js'), + progressStream = require('progress-stream'), + safe = require('safetydance'), + services = require('./services.js'), + settings = require('./settings.js'), + shell = require('./shell.js'), + storage = require('./storage.js'), + syncer = require('./syncer.js'), + tar = require('tar-fs'), + TransformStream = require('stream').Transform, + zlib = require('zlib'), + util = require('util'); + +const BACKUP_UPLOAD_CMD = path.join(__dirname, 'scripts/backupupload.js'); + +function debugApp(app) { + assert(typeof app === 'object'); + + debug(app.fqdn + ' ' + util.format.apply(util, Array.prototype.slice.call(arguments, 1))); +} + +function canBackupApp(app) { + // only backup apps that are installed or specific pending states + + // stopped apps cannot be backed up because addons might be down (redis) + if (app.runState === apps.RSTATE_STOPPED) return false; + + // we used to check the health here but that doesn't work for stopped apps. it's better to just fail + // and inform the user if the backup fails and the app addons have not been setup yet. + return app.installationState === apps.ISTATE_INSTALLED || + app.installationState === apps.ISTATE_PENDING_CONFIGURE || + app.installationState === apps.ISTATE_PENDING_BACKUP || // called from apptask + app.installationState === apps.ISTATE_PENDING_UPDATE; // called from apptask +} + +function encryptFilePath(filePath, encryption) { + assert.strictEqual(typeof filePath, 'string'); + assert.strictEqual(typeof encryption, 'object'); + + var encryptedParts = filePath.split('/').map(function (part) { + let hmac = crypto.createHmac('sha256', Buffer.from(encryption.filenameHmacKey, 'hex')); + const iv = hmac.update(part).digest().slice(0, 16); // iv has to be deterministic, for our sync (copy) logic to work + const cipher = crypto.createCipheriv('aes-256-cbc', Buffer.from(encryption.filenameKey, 'hex'), iv); + let crypt = cipher.update(part); + crypt = Buffer.concat([ iv, crypt, cipher.final() ]); + + return crypt.toString('base64') // ensures path is valid + .replace(/\//g, '-') // replace '/' of base64 since it conflicts with path separator + .replace(/=/g,''); // strip trailing = padding. this is only needed if we concat base64 strings, which we don't + }); + + return encryptedParts.join('/'); +} + +function decryptFilePath(filePath, encryption) { + assert.strictEqual(typeof filePath, 'string'); + assert.strictEqual(typeof encryption, 'object'); + + let decryptedParts = []; + for (let part of filePath.split('/')) { + part = part + Array(part.length % 4).join('='); // add back = padding + part = part.replace(/-/g, '/'); // replace with '/' + + try { + const buffer = Buffer.from(part, 'base64'); + const iv = buffer.slice(0, 16); + let decrypt = crypto.createDecipheriv('aes-256-cbc', Buffer.from(encryption.filenameKey, 'hex'), iv); + const plainText = decrypt.update(buffer.slice(16)); + const plainTextString = Buffer.concat([ plainText, decrypt.final() ]).toString('utf8'); + const hmac = crypto.createHmac('sha256', Buffer.from(encryption.filenameHmacKey, 'hex')); + if (!hmac.update(plainTextString).digest().slice(0, 16).equals(iv)) return { error: new BoxError(BoxError.CRYPTO_ERROR, `mac error decrypting part ${part} of path ${filePath}`) }; + + decryptedParts.push(plainTextString); + } catch (error) { + debug(`Error decrypting part ${part} of path ${filePath}:`, error); + return { error: new BoxError(BoxError.CRYPTO_ERROR, `Error decrypting part ${part} of path ${filePath}: ${error.message}`) }; + } + } + + return { result: decryptedParts.join('/') }; +} + +class EncryptStream extends TransformStream { + constructor(encryption) { + super(); + this._headerPushed = false; + this._iv = crypto.randomBytes(16); + this._cipher = crypto.createCipheriv('aes-256-cbc', Buffer.from(encryption.dataKey, 'hex'), this._iv); + this._hmac = crypto.createHmac('sha256', Buffer.from(encryption.dataHmacKey, 'hex')); + } + + pushHeaderIfNeeded() { + if (!this._headerPushed) { + const magic = Buffer.from('CBV2'); + this.push(magic); + this._hmac.update(magic); + this.push(this._iv); + this._hmac.update(this._iv); + this._headerPushed = true; + } + } + + _transform(chunk, ignoredEncoding, callback) { + this.pushHeaderIfNeeded(); + + try { + const crypt = this._cipher.update(chunk); + this._hmac.update(crypt); + callback(null, crypt); + } catch (error) { + callback(error); + } + } + + _flush(callback) { + try { + this.pushHeaderIfNeeded(); // for 0-length files + const crypt = this._cipher.final(); + this.push(crypt); + this._hmac.update(crypt); + callback(null, this._hmac.digest()); // +32 bytes + } catch (error) { + callback(error); + } + } +} + +class DecryptStream extends TransformStream { + constructor(encryption) { + super(); + this._key = Buffer.from(encryption.dataKey, 'hex'); + this._header = Buffer.alloc(0); + this._decipher = null; + this._hmac = crypto.createHmac('sha256', Buffer.from(encryption.dataHmacKey, 'hex')); + this._buffer = Buffer.alloc(0); + } + + _transform(chunk, ignoredEncoding, callback) { + const needed = 20 - this._header.length; // 4 for magic, 16 for iv + + if (this._header.length !== 20) { // not gotten header yet + this._header = Buffer.concat([this._header, chunk.slice(0, needed)]); + if (this._header.length !== 20) return callback(); + + if (!this._header.slice(0, 4).equals(new Buffer.from('CBV2'))) return callback(new BoxError(BoxError.CRYPTO_ERROR, 'Invalid magic in header')); + + const iv = this._header.slice(4); + this._decipher = crypto.createDecipheriv('aes-256-cbc', this._key, iv); + this._hmac.update(this._header); + } + + this._buffer = Buffer.concat([ this._buffer, chunk.slice(needed) ]); + if (this._buffer.length < 32) return callback(); // hmac trailer length is 32 + + try { + const cipherText = this._buffer.slice(0, -32); + this._hmac.update(cipherText); + const plainText = this._decipher.update(cipherText); + this._buffer = this._buffer.slice(-32); + callback(null, plainText); + } catch (error) { + callback(error); + } + } + + _flush (callback) { + if (this._buffer.length !== 32) return callback(new BoxError(BoxError.CRYPTO_ERROR, 'Invalid password or tampered file (not enough data)')); + + try { + if (!this._hmac.digest().equals(this._buffer)) return callback(new BoxError(BoxError.CRYPTO_ERROR, 'Invalid password or tampered file (mac mismatch)')); + + const plainText = this._decipher.final(); + callback(null, plainText); + } catch (error) { + callback(error); + } + } +} + +function createReadStream(sourceFile, encryption) { + assert.strictEqual(typeof sourceFile, 'string'); + assert.strictEqual(typeof encryption, 'object'); + + var stream = fs.createReadStream(sourceFile); + var ps = progressStream({ time: 10000 }); // display a progress every 10 seconds + + stream.on('error', function (error) { + debug(`createReadStream: read stream error at ${sourceFile}`, error); + ps.emit('error', new BoxError(BoxError.FS_ERROR, `Error reading ${sourceFile}: ${error.message} ${error.code}`)); + }); + + stream.on('open', () => ps.emit('open')); + + if (encryption) { + let encryptStream = new EncryptStream(encryption); + + encryptStream.on('error', function (error) { + debug(`createReadStream: encrypt stream error ${sourceFile}`, error); + ps.emit('error', new BoxError(BoxError.CRYPTO_ERROR, `Encryption error at ${sourceFile}: ${error.message}`)); + }); + + return stream.pipe(encryptStream).pipe(ps); + } else { + return stream.pipe(ps); + } +} + +function createWriteStream(destFile, encryption) { + assert.strictEqual(typeof destFile, 'string'); + assert.strictEqual(typeof encryption, 'object'); + + var stream = fs.createWriteStream(destFile); + var ps = progressStream({ time: 10000 }); // display a progress every 10 seconds + + stream.on('error', function (error) { + debug(`createWriteStream: write stream error ${destFile}`, error); + ps.emit('error', new BoxError(BoxError.FS_ERROR, `Write error ${destFile}: ${error.message}`)); + }); + + stream.on('finish', function () { + debug('createWriteStream: done.'); + // we use a separate event because ps is a through2 stream which emits 'finish' event indicating end of inStream and not write + ps.emit('done'); + }); + + if (encryption) { + let decrypt = new DecryptStream(encryption); + decrypt.on('error', function (error) { + debug(`createWriteStream: decrypt stream error ${destFile}`, error); + ps.emit('error', new BoxError(BoxError.CRYPTO_ERROR, `Decryption error at ${destFile}: ${error.message}`)); + }); + + ps.pipe(decrypt).pipe(stream); + } else { + ps.pipe(stream); + } + + return ps; +} + +function tarPack(dataLayout, encryption, callback) { + assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout'); + assert.strictEqual(typeof encryption, 'object'); + assert.strictEqual(typeof callback, 'function'); + + var pack = tar.pack('/', { + dereference: false, // pack the symlink and not what it points to + entries: dataLayout.localPaths(), + ignoreStatError: (path, err) => { + debug(`tarPack: error stat'ing ${path} - ${err.code}`); + return err.code === 'ENOENT'; // ignore if file or dir got removed (probably some temporary file) + }, + map: function(header) { + header.name = dataLayout.toRemotePath(header.name); + // the tar pax format allows us to encode filenames > 100 and size > 8GB (see #640) + // https://www.systutorials.com/docs/linux/man/5-star/ + if (header.size > 8589934590 || header.name > 99) header.pax = { size: header.size }; + return header; + }, + strict: false // do not error for unknown types (skip fifo, char/block devices) + }); + + var gzip = zlib.createGzip({}); + var ps = progressStream({ time: 10000 }); // emit 'progress' every 10 seconds + + pack.on('error', function (error) { + debug('tarPack: tar stream error.', error); + ps.emit('error', new BoxError(BoxError.EXTERNAL_ERROR, error.message)); + }); + + gzip.on('error', function (error) { + debug('tarPack: gzip stream error.', error); + ps.emit('error', new BoxError(BoxError.EXTERNAL_ERROR, error.message)); + }); + + if (encryption) { + const encryptStream = new EncryptStream(encryption); + encryptStream.on('error', function (error) { + debug('tarPack: encrypt stream error.', error); + ps.emit('error', new BoxError(BoxError.EXTERNAL_ERROR, error.message)); + }); + + pack.pipe(gzip).pipe(encryptStream).pipe(ps); + } else { + pack.pipe(gzip).pipe(ps); + } + + return callback(null, ps); +} + +function sync(backupConfig, backupId, dataLayout, progressCallback, callback) { + assert.strictEqual(typeof backupConfig, 'object'); + assert.strictEqual(typeof backupId, 'string'); + assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout'); + assert.strictEqual(typeof progressCallback, 'function'); + assert.strictEqual(typeof callback, 'function'); + + // the number here has to take into account the s3.upload partSize (which is 10MB). So 20=200MB + const concurrency = backupConfig.syncConcurrency || (backupConfig.provider === 's3' ? 20 : 10); + + syncer.sync(dataLayout, function processTask(task, iteratorCallback) { + debug('sync: processing task: %j', task); + // the empty task.path is special to signify the directory + const destPath = task.path && backupConfig.encryption ? encryptFilePath(task.path, backupConfig.encryption) : task.path; + const backupFilePath = path.join(storage.getBackupFilePath(backupConfig, backupId, backupConfig.format), destPath); + + if (task.operation === 'removedir') { + debug(`Removing directory ${backupFilePath}`); + return storage.api(backupConfig.provider).removeDir(backupConfig, backupFilePath) + .on('progress', (message) => progressCallback({ message })) + .on('done', iteratorCallback); + } else if (task.operation === 'remove') { + debug(`Removing ${backupFilePath}`); + return storage.api(backupConfig.provider).remove(backupConfig, backupFilePath, iteratorCallback); + } + + var retryCount = 0; + async.retry({ times: 5, interval: 20000 }, function (retryCallback) { + retryCallback = once(retryCallback); // protect again upload() erroring much later after read stream error + + ++retryCount; + if (task.operation === 'add') { + progressCallback({ message: `Adding ${task.path}` + (retryCount > 1 ? ` (Try ${retryCount})` : '') }); + debug(`Adding ${task.path} position ${task.position} try ${retryCount}`); + var stream = createReadStream(dataLayout.toLocalPath('./' + task.path), backupConfig.encryption); + stream.on('error', (error) => retryCallback(error.message.includes('ENOENT') ? null : error)); // ignore error if file disappears + stream.on('progress', function (progress) { + const transferred = Math.round(progress.transferred/1024/1024), speed = Math.round(progress.speed/1024/1024); + if (!transferred && !speed) return progressCallback({ message: `Uploading ${task.path}` }); // 0M@0MBps looks wrong + progressCallback({ message: `Uploading ${task.path}: ${transferred}M@${speed}MBps` }); // 0M@0MBps looks wrong + }); + // only create the destination path when we have confirmation that the source is available. otherwise, we end up with + // files owned as 'root' and the cp later will fail + stream.on('open', function () { + storage.api(backupConfig.provider).upload(backupConfig, backupFilePath, stream, function (error) { + debug(error ? `Error uploading ${task.path} try ${retryCount}: ${error.message}` : `Uploaded ${task.path}`); + retryCallback(error); + }); + }); + } + }, iteratorCallback); + }, concurrency, function (error) { + if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, error.message)); + + callback(); + }); +} + +// this is not part of 'snapshotting' because we need root access to traverse +function saveFsMetadata(dataLayout, metadataFile, callback) { + assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout'); + assert.strictEqual(typeof metadataFile, 'string'); + assert.strictEqual(typeof callback, 'function'); + + // contains paths prefixed with './' + let metadata = { + emptyDirs: [], + execFiles: [], + symlinks: [] + }; + + // we assume small number of files. spawnSync will raise a ENOBUFS error after maxBuffer + for (let lp of dataLayout.localPaths()) { + const emptyDirs = safe.child_process.execSync(`find ${lp} -type d -empty`, { encoding: 'utf8', maxBuffer: 1024 * 1024 * 30 }); + if (emptyDirs === null) return callback(new BoxError(BoxError.FS_ERROR, `Error finding empty dirs: ${safe.error.message}`)); + if (emptyDirs.length) metadata.emptyDirs = metadata.emptyDirs.concat(emptyDirs.trim().split('\n').map((ed) => dataLayout.toRemotePath(ed))); + + const execFiles = safe.child_process.execSync(`find ${lp} -type f -executable`, { encoding: 'utf8', maxBuffer: 1024 * 1024 * 30 }); + if (execFiles === null) return callback(new BoxError(BoxError.FS_ERROR, `Error finding executables: ${safe.error.message}`)); + if (execFiles.length) metadata.execFiles = metadata.execFiles.concat(execFiles.trim().split('\n').map((ef) => dataLayout.toRemotePath(ef))); + + const symlinks = safe.child_process.execSync(`find ${lp} -type l`, { encoding: 'utf8', maxBuffer: 1024 * 1024 * 30 }); + if (symlinks === null) return callback(new BoxError(BoxError.FS_ERROR, `Error finding symlinks: ${safe.error.message}`)); + if (symlinks.length) metadata.symlinks = metadata.symlinks.concat(symlinks.trim().split('\n').map((sl) => { + const target = safe.fs.readlinkSync(sl); + return { path: dataLayout.toRemotePath(sl), target }; + })); + } + + if (!safe.fs.writeFileSync(metadataFile, JSON.stringify(metadata, null, 4))) return callback(new BoxError(BoxError.FS_ERROR, `Error writing fs metadata: ${safe.error.message}`)); + + callback(); +} + +// this function is called via backupupload (since it needs root to traverse app's directory) +function upload(backupId, format, dataLayoutString, progressCallback, callback) { + assert.strictEqual(typeof backupId, 'string'); + assert.strictEqual(typeof format, 'string'); + assert.strictEqual(typeof dataLayoutString, 'string'); + assert.strictEqual(typeof progressCallback, 'function'); + assert.strictEqual(typeof callback, 'function'); + + debug(`upload: id ${backupId} format ${format} dataLayout ${dataLayoutString}`); + + const dataLayout = DataLayout.fromString(dataLayoutString); + + settings.getBackupConfig(function (error, backupConfig) { + if (error) return callback(error); + + storage.api(backupConfig.provider).checkPreconditions(backupConfig, dataLayout, function (error) { + if (error) return callback(error); + + if (format === 'tgz') { + async.retry({ times: 5, interval: 20000 }, function (retryCallback) { + retryCallback = once(retryCallback); // protect again upload() erroring much later after tar stream error + + tarPack(dataLayout, backupConfig.encryption, function (error, tarStream) { + if (error) return retryCallback(error); + + tarStream.on('progress', function (progress) { + const transferred = Math.round(progress.transferred/1024/1024), speed = Math.round(progress.speed/1024/1024); + if (!transferred && !speed) return progressCallback({ message: 'Uploading backup' }); // 0M@0MBps looks wrong + progressCallback({ message: `Uploading backup ${transferred}M@${speed}MBps` }); + }); + tarStream.on('error', retryCallback); // already returns BoxError + + storage.api(backupConfig.provider).upload(backupConfig, storage.getBackupFilePath(backupConfig, backupId, format), tarStream, retryCallback); + }); + }, callback); + } else { + async.series([ + saveFsMetadata.bind(null, dataLayout, `${dataLayout.localRoot()}/fsmetadata.json`), + sync.bind(null, backupConfig, backupId, dataLayout, progressCallback) + ], callback); + } + }); + }); +} + +function tarExtract(inStream, dataLayout, encryption, callback) { + assert.strictEqual(typeof inStream, 'object'); + assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout'); + assert.strictEqual(typeof encryption, 'object'); + assert.strictEqual(typeof callback, 'function'); + + var gunzip = zlib.createGunzip({}); + var ps = progressStream({ time: 10000 }); // display a progress every 10 seconds + var extract = tar.extract('/', { + map: function (header) { + header.name = dataLayout.toLocalPath(header.name); + return header; + }, + dmode: 500 // ensure directory is writable + }); + + const emitError = once((error) => { + inStream.destroy(); + ps.emit('error', error); + }); + + inStream.on('error', function (error) { + debug('tarExtract: input stream error.', error); + emitError(new BoxError(BoxError.EXTERNAL_ERROR, error.message)); + }); + + gunzip.on('error', function (error) { + debug('tarExtract: gunzip stream error.', error); + emitError(new BoxError(BoxError.EXTERNAL_ERROR, error.message)); + }); + + extract.on('error', function (error) { + debug('tarExtract: extract stream error.', error); + emitError(new BoxError(BoxError.EXTERNAL_ERROR, error.message)); + }); + + extract.on('finish', function () { + debug('tarExtract: done.'); + // we use a separate event because ps is a through2 stream which emits 'finish' event indicating end of inStream and not extract + ps.emit('done'); + }); + + if (encryption) { + let decrypt = new DecryptStream(encryption); + decrypt.on('error', function (error) { + debug('tarExtract: decrypt stream error.', error); + emitError(new BoxError(BoxError.EXTERNAL_ERROR, `Failed to decrypt: ${error.message}`)); + }); + inStream.pipe(ps).pipe(decrypt).pipe(gunzip).pipe(extract); + } else { + inStream.pipe(ps).pipe(gunzip).pipe(extract); + } + + callback(null, ps); +} + +function restoreFsMetadata(dataLayout, metadataFile, callback) { + assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout'); + assert.strictEqual(typeof metadataFile, 'string'); + assert.strictEqual(typeof callback, 'function'); + + debug(`Recreating empty directories in ${dataLayout.toString()}`); + + var metadataJson = safe.fs.readFileSync(metadataFile, 'utf8'); + if (metadataJson === null) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Error loading fsmetadata.json:' + safe.error.message)); + var metadata = safe.JSON.parse(metadataJson); + if (metadata === null) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Error parsing fsmetadata.json:' + safe.error.message)); + + async.eachSeries(metadata.emptyDirs, function createPath(emptyDir, iteratorDone) { + fs.mkdir(dataLayout.toLocalPath(emptyDir), { recursive: true }, iteratorDone); + }, function (error) { + if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `unable to create path: ${error.message}`)); + + async.eachSeries(metadata.execFiles, function createPath(execFile, iteratorDone) { + fs.chmod(dataLayout.toLocalPath(execFile), parseInt('0755', 8), iteratorDone); + }, function (error) { + if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `unable to chmod: ${error.message}`)); + + async.eachSeries(metadata.symlinks || [], function createSymlink(symlink, iteratorDone) { + if (!symlink.target) return iteratorDone(); + // the path may not exist if we had a directory full of symlinks + fs.mkdir(path.dirname(dataLayout.toLocalPath(symlink.path)), { recursive: true }, function (error) { + if (error) return iteratorDone(error); + + fs.symlink(symlink.target, dataLayout.toLocalPath(symlink.path), 'file', iteratorDone); + }); + }, function (error) { + if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `unable to symlink: ${error.message}`)); + + callback(); + }); + }); + }); +} + +function downloadDir(backupConfig, backupFilePath, dataLayout, progressCallback, callback) { + assert.strictEqual(typeof backupConfig, 'object'); + assert.strictEqual(typeof backupFilePath, 'string'); + assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout'); + assert.strictEqual(typeof progressCallback, 'function'); + assert.strictEqual(typeof callback, 'function'); + + debug(`downloadDir: ${backupFilePath} to ${dataLayout.toString()}`); + + function downloadFile(entry, done) { + let relativePath = path.relative(backupFilePath, entry.fullPath); + if (backupConfig.encryption) { + const { error, result } = decryptFilePath(relativePath, backupConfig.encryption); + if (error) return done(new BoxError(BoxError.CRYPTO_ERROR, 'Unable to decrypt file')); + relativePath = result; + } + const destFilePath = dataLayout.toLocalPath('./' + relativePath); + + fs.mkdir(path.dirname(destFilePath), { recursive: true }, function (error) { + if (error) return done(new BoxError(BoxError.FS_ERROR, error.message)); + + async.retry({ times: 5, interval: 20000 }, function (retryCallback) { + storage.api(backupConfig.provider).download(backupConfig, entry.fullPath, function (error, sourceStream) { + if (error) { + progressCallback({ message: `Download ${entry.fullPath} to ${destFilePath} errored: ${error.message}` }); + return retryCallback(error); + } + + let destStream = createWriteStream(destFilePath, backupConfig.encryption); + + // protect against multiple errors. must destroy the write stream so that a previous retry does not write + let closeAndRetry = once((error) => { + if (error) progressCallback({ message: `Download ${entry.fullPath} to ${destFilePath} errored: ${error.message}` }); + else progressCallback({ message: `Download ${entry.fullPath} to ${destFilePath} finished` }); + sourceStream.destroy(); + destStream.destroy(); + retryCallback(error); + }); + + destStream.on('progress', function (progress) { + const transferred = Math.round(progress.transferred/1024/1024), speed = Math.round(progress.speed/1024/1024); + if (!transferred && !speed) return progressCallback({ message: `Downloading ${entry.fullPath}` }); // 0M@0MBps looks wrong + progressCallback({ message: `Downloading ${entry.fullPath}: ${transferred}M@${speed}MBps` }); + }); + destStream.on('error', closeAndRetry); + + sourceStream.on('error', closeAndRetry); + + progressCallback({ message: `Downloading ${entry.fullPath} to ${destFilePath}` }); + + sourceStream.pipe(destStream, { end: true }).on('done', closeAndRetry); + }); + }, done); + }); + } + + storage.api(backupConfig.provider).listDir(backupConfig, backupFilePath, 1000, function (entries, iteratorDone) { + // https://www.digitalocean.com/community/questions/rate-limiting-on-spaces?answer=40441 + const concurrency = backupConfig.downloadConcurrency || (backupConfig.provider === 's3' ? 30 : 10); + + async.eachLimit(entries, concurrency, downloadFile, iteratorDone); + }, callback); +} + +function download(backupConfig, backupId, format, dataLayout, progressCallback, callback) { + assert.strictEqual(typeof backupConfig, 'object'); + assert.strictEqual(typeof backupId, 'string'); + assert.strictEqual(typeof format, 'string'); + assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout'); + assert.strictEqual(typeof progressCallback, 'function'); + assert.strictEqual(typeof callback, 'function'); + + debug(`download: Downloading ${backupId} of format ${format} to ${dataLayout.toString()}`); + + const backupFilePath = storage.getBackupFilePath(backupConfig, backupId, format); + + if (format === 'tgz') { + async.retry({ times: 5, interval: 20000 }, function (retryCallback) { + storage.api(backupConfig.provider).download(backupConfig, backupFilePath, function (error, sourceStream) { + if (error) return retryCallback(error); + + tarExtract(sourceStream, dataLayout, backupConfig.encryption, function (error, ps) { + if (error) return retryCallback(error); + + ps.on('progress', function (progress) { + const transferred = Math.round(progress.transferred/1024/1024), speed = Math.round(progress.speed/1024/1024); + if (!transferred && !speed) return progressCallback({ message: 'Downloading backup' }); // 0M@0MBps looks wrong + progressCallback({ message: `Downloading ${transferred}M@${speed}MBps` }); + }); + ps.on('error', retryCallback); + ps.on('done', retryCallback); + }); + }); + }, callback); + } else { + downloadDir(backupConfig, backupFilePath, dataLayout, progressCallback, function (error) { + if (error) return callback(error); + + restoreFsMetadata(dataLayout, `${dataLayout.localRoot()}/fsmetadata.json`, callback); + }); + } +} + +function restore(backupConfig, backupId, progressCallback, callback) { + assert.strictEqual(typeof backupConfig, 'object'); + assert.strictEqual(typeof backupId, 'string'); + assert.strictEqual(typeof progressCallback, 'function'); + assert.strictEqual(typeof callback, 'function'); + + const boxDataDir = safe.fs.realpathSync(paths.BOX_DATA_DIR); + if (!boxDataDir) return callback(new BoxError(BoxError.FS_ERROR, `Error resolving boxdata: ${safe.error.message}`)); + const dataLayout = new DataLayout(boxDataDir, []); + + download(backupConfig, backupId, backupConfig.format, dataLayout, progressCallback, function (error) { + if (error) return callback(error); + + debug('restore: download completed, importing database'); + + database.importFromFile(`${dataLayout.localRoot()}/box.mysqldump`, function (error) { + if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error)); + + debug('restore: database imported'); + + settings.initCache(callback); + }); + }); +} + +function downloadApp(app, restoreConfig, progressCallback, callback) { + assert.strictEqual(typeof app, 'object'); + assert.strictEqual(typeof restoreConfig, 'object'); + assert.strictEqual(typeof progressCallback, 'function'); + assert.strictEqual(typeof callback, 'function'); + + const appDataDir = safe.fs.realpathSync(path.join(paths.APPS_DATA_DIR, app.id)); + if (!appDataDir) return callback(new BoxError(BoxError.FS_ERROR, safe.error.message)); + const dataLayout = new DataLayout(appDataDir, app.dataDir ? [{ localDir: app.dataDir, remoteDir: 'data' }] : []); + + const startTime = new Date(); + const getBackupConfigFunc = restoreConfig.backupConfig ? (next) => next(null, restoreConfig.backupConfig) : settings.getBackupConfig; + + getBackupConfigFunc(function (error, backupConfig) { + if (error) return callback(error); + + download(backupConfig, restoreConfig.backupId, restoreConfig.backupFormat, dataLayout, progressCallback, function (error) { + debug('downloadApp: time: %s', (new Date() - startTime)/1000); + + callback(error); + }); + }); +} + +function runBackupUpload(uploadConfig, progressCallback, callback) { + assert.strictEqual(typeof uploadConfig, 'object'); + assert.strictEqual(typeof progressCallback, 'function'); + assert.strictEqual(typeof callback, 'function'); + + const { backupId, backupConfig, dataLayout, progressTag } = uploadConfig; + assert.strictEqual(typeof backupId, 'string'); + assert.strictEqual(typeof backupConfig, 'object'); + assert.strictEqual(typeof progressTag, 'string'); + assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout'); + + let result = ''; // the script communicates error result as a string + + // https://stackoverflow.com/questions/48387040/node-js-recommended-max-old-space-size + const envCopy = Object.assign({}, process.env); + if (backupConfig.memoryLimit && backupConfig.memoryLimit >= 2*1024*1024*1024) { + const heapSize = Math.min((backupConfig.memoryLimit/1024/1024) - 256, 8192); + debug(`runBackupUpload: adjusting heap size to ${heapSize}M`); + envCopy.NODE_OPTIONS = `--max-old-space-size=${heapSize}`; + } + + shell.sudo(`backup-${backupId}`, [ BACKUP_UPLOAD_CMD, backupId, backupConfig.format, dataLayout.toString() ], { env: envCopy, preserveEnv: true, ipc: true }, function (error) { + if (error && (error.code === null /* signal */ || (error.code !== 0 && error.code !== 50))) { // backuptask crashed + return callback(new BoxError(BoxError.INTERNAL_ERROR, 'Backuptask crashed')); + } else if (error && error.code === 50) { // exited with error + return callback(new BoxError(BoxError.EXTERNAL_ERROR, result)); + } + + callback(); + }).on('message', function (progress) { // this is { message } or { result } + if ('message' in progress) return progressCallback({ message: `${progress.message} (${progressTag})` }); + debug(`runBackupUpload: result - ${JSON.stringify(progress)}`); + result = progress.result; + }); +} + +function snapshotBox(progressCallback, callback) { + assert.strictEqual(typeof progressCallback, 'function'); + assert.strictEqual(typeof callback, 'function'); + + progressCallback({ message: 'Snapshotting box' }); + + const startTime = new Date(); + + database.exportToFile(`${paths.BOX_DATA_DIR}/box.mysqldump`, function (error) { + if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error)); + + debug(`snapshotBox: took ${(new Date() - startTime)/1000} seconds`); + + return callback(); + }); +} + +function uploadBoxSnapshot(backupConfig, progressCallback, callback) { + assert.strictEqual(typeof backupConfig, 'object'); + assert.strictEqual(typeof progressCallback, 'function'); + assert.strictEqual(typeof callback, 'function'); + + snapshotBox(progressCallback, function (error) { + if (error) return callback(error); + + const boxDataDir = safe.fs.realpathSync(paths.BOX_DATA_DIR); + if (!boxDataDir) return callback(new BoxError(BoxError.FS_ERROR, `Error resolving boxdata: ${safe.error.message}`)); + + const uploadConfig = { + backupId: 'snapshot/box', + backupConfig, + dataLayout: new DataLayout(boxDataDir, []), + progressTag: 'box' + }; + + progressCallback({ message: 'Uploading box snapshot' }); + + const startTime = new Date(); + + runBackupUpload(uploadConfig, progressCallback, function (error) { + if (error) return callback(error); + + debug(`uploadBoxSnapshot: took ${(new Date() - startTime)/1000} seconds`); + + backups.setSnapshotInfo('box', { timestamp: new Date().toISOString(), format: backupConfig.format }, callback); + }); + }); +} + +async function rotateBoxBackup(backupConfig, tag, options, appBackupIds, progressCallback) { + assert.strictEqual(typeof backupConfig, 'object'); + assert.strictEqual(typeof tag, 'string'); + assert.strictEqual(typeof options, 'object'); + assert(Array.isArray(appBackupIds)); + assert.strictEqual(typeof progressCallback, 'function'); + + const backupId = `${tag}/box_v${constants.VERSION}`; + const format = backupConfig.format; + + debug(`Rotating box backup to id ${backupId}`); + + const data = { + encryptionVersion: backupConfig.encryption ? 2 : null, + packageVersion: constants.VERSION, + type: backups.BACKUP_TYPE_BOX, + state: backups.BACKUP_STATE_CREATING, + identifier: 'box', + dependsOn: appBackupIds, + manifest: null, + format: format + }; + + await backups.add(backupId, data); + + return new Promise((resolve, reject) => { + const copy = storage.api(backupConfig.provider).copy(backupConfig, storage.getBackupFilePath(backupConfig, 'snapshot/box', format), storage.getBackupFilePath(backupConfig, backupId, format)); + copy.on('progress', (message) => progressCallback({ message: `box: ${message}` })); + copy.on('done', async function (copyBackupError) { + const state = copyBackupError ? backups.BACKUP_STATE_ERROR : backups.BACKUP_STATE_NORMAL; + + const [error] = await safe(backups.update(backupId, { preserveSecs: options.preserveSecs || 0, state })); + if (copyBackupError) return reject(copyBackupError); + if (error) return reject(error); + + debug(`Rotated box backup successfully as id ${backupId}`); + + resolve(backupId); + }); + }); +} + +function backupBoxWithAppBackupIds(appBackupIds, tag, options, progressCallback, callback) { + assert(Array.isArray(appBackupIds)); + assert.strictEqual(typeof tag, 'string'); + assert.strictEqual(typeof options, 'object'); + assert.strictEqual(typeof progressCallback, 'function'); + assert.strictEqual(typeof callback, 'function'); + + settings.getBackupConfig(function (error, backupConfig) { + if (error) return callback(error); + + uploadBoxSnapshot(backupConfig, progressCallback, async function (error) { + if (error) return callback(error); + + const [rotateError] = await safe(rotateBoxBackup(backupConfig, tag, options, appBackupIds, progressCallback)); + callback(rotateError); + }); + }); +} + +async function rotateAppBackup(backupConfig, app, tag, options, progressCallback) { + assert.strictEqual(typeof backupConfig, 'object'); + assert.strictEqual(typeof app, 'object'); + assert.strictEqual(typeof tag, 'string'); + assert.strictEqual(typeof options, 'object'); + assert.strictEqual(typeof progressCallback, 'function'); + + const startTime = new Date(); + + const snapshotInfo = backups.getSnapshotInfo(app.id); + + const manifest = snapshotInfo.restoreConfig ? snapshotInfo.restoreConfig.manifest : snapshotInfo.manifest; // compat + const backupId = `${tag}/app_${app.fqdn}_v${manifest.version}`; + const format = backupConfig.format; + + debug(`Rotating app backup of ${app.id} to id ${backupId}`); + + const data = { + encryptionVersion: backupConfig.encryption ? 2 : null, + packageVersion: manifest.version, + type: backups.BACKUP_TYPE_APP, + state: backups.BACKUP_STATE_CREATING, + identifier: app.id, + dependsOn: [ ], + manifest, + format: format + }; + + await backups.add(backupId, data); + + return new Promise((resolve, reject) => { + const copy = storage.api(backupConfig.provider).copy(backupConfig, storage.getBackupFilePath(backupConfig, `snapshot/app_${app.id}`, format), storage.getBackupFilePath(backupConfig, backupId, format)); + copy.on('progress', (message) => progressCallback({ message: `${message} (${app.fqdn})` })); + copy.on('done', async function (copyBackupError) { + const state = copyBackupError ? backups.BACKUP_STATE_ERROR : backups.BACKUP_STATE_NORMAL; + + const [error] = await safe(backups.update(backupId, { preserveSecs: options.preserveSecs || 0, state })); + if (copyBackupError) return reject(copyBackupError); + if (error) return reject(error); + + debug(`Rotated app backup of ${app.id} successfully to id ${backupId}. Took ${(new Date() - startTime)/1000} seconds`); + + resolve(backupId); + }); + }); +} + +function backupApp(app, options, progressCallback, callback) { + assert.strictEqual(typeof app, 'object'); + assert.strictEqual(typeof options, 'object'); + assert.strictEqual(typeof progressCallback, 'function'); + assert.strictEqual(typeof callback, 'function'); + + if (options.snapshotOnly) return snapshotApp(app, progressCallback, callback); + + const tag = (new Date()).toISOString().replace(/[T.]/g, '-').replace(/[:Z]/g,''); + + debug(`backupApp - Backing up ${app.fqdn} with tag ${tag}`); + + backupAppWithTag(app, tag, options, progressCallback, callback); +} + + +function snapshotApp(app, progressCallback, callback) { + assert.strictEqual(typeof app, 'object'); + assert.strictEqual(typeof progressCallback, 'function'); + assert.strictEqual(typeof callback, 'function'); + + const startTime = new Date(); + progressCallback({ message: `Snapshotting app ${app.fqdn}` }); + + apps.backupConfig(app, function (error) { + if (error) return callback(error); + + services.backupAddons(app, app.manifest.addons, function (error) { + if (error) return callback(error); + + debugApp(app, `snapshotApp: took ${(new Date() - startTime)/1000} seconds`); + + return callback(null); + }); + }); +} + +function uploadAppSnapshot(backupConfig, app, progressCallback, callback) { + assert.strictEqual(typeof backupConfig, 'object'); + assert.strictEqual(typeof app, 'object'); + assert.strictEqual(typeof progressCallback, 'function'); + assert.strictEqual(typeof callback, 'function'); + + snapshotApp(app, progressCallback, function (error) { + if (error) return callback(error); + + const backupId = util.format('snapshot/app_%s', app.id); + const appDataDir = safe.fs.realpathSync(path.join(paths.APPS_DATA_DIR, app.id)); + if (!appDataDir) return callback(new BoxError(BoxError.FS_ERROR, `Error resolving appsdata: ${safe.error.message}`)); + + const dataLayout = new DataLayout(appDataDir, app.dataDir ? [{ localDir: app.dataDir, remoteDir: 'data' }] : []); + + progressCallback({ message: `Uploading app snapshot ${app.fqdn}`}); + + const uploadConfig = { + backupId, + backupConfig, + dataLayout, + progressTag: app.fqdn + }; + + const startTime = new Date(); + + runBackupUpload(uploadConfig, progressCallback, function (error) { + if (error) return callback(error); + + debugApp(app, `uploadAppSnapshot: ${backupId} done. ${(new Date() - startTime)/1000} seconds`); + + backups.setSnapshotInfo(app.id, { timestamp: new Date().toISOString(), manifest: app.manifest, format: backupConfig.format }, callback); + }); + }); +} + +async function backupAppWithTag(app, tag, options, progressCallback, callback) { + assert.strictEqual(typeof app, 'object'); + assert.strictEqual(typeof tag, 'string'); + assert.strictEqual(typeof options, 'object'); + assert.strictEqual(typeof progressCallback, 'function'); + assert.strictEqual(typeof callback, 'function'); + + if (!canBackupApp(app)) { // if we cannot backup, reuse it's most recent backup + const [error, results] = await safe(backups.getByIdentifierAndStatePaged(app.id, backups.BACKUP_STATE_NORMAL, 1, 1)); + if (error) return callback(error); + if (results.length === 0) return callback(null, null); // no backup to re-use + + return callback(null, results[0].id); + } + + settings.getBackupConfig(function (error, backupConfig) { + if (error) return callback(error); + + uploadAppSnapshot(backupConfig, app, progressCallback, async function (error) { + if (error) return callback(error); + + const [rotateError] = await safe(rotateAppBackup(backupConfig, app, tag, options, progressCallback)); + callback(rotateError); + }); + }); +} + +// this function expects you to have a lock. Unlike other progressCallback this also has a progress field +function backupBoxAndApps(options, progressCallback, callback) { + assert.strictEqual(typeof options, 'object'); + assert.strictEqual(typeof progressCallback, 'function'); + assert.strictEqual(typeof callback, 'function'); + + const tag = (new Date()).toISOString().replace(/[T.]/g, '-').replace(/[:Z]/g,''); + + apps.getAll(function (error, allApps) { + if (error) return callback(error); + + let percent = 1; + let step = 100/(allApps.length+2); + + async.mapSeries(allApps, function iterator(app, iteratorCallback) { + progressCallback({ percent: percent, message: `Backing up ${app.fqdn}` }); + percent += step; + + if (!app.enableBackup) { + debug(`Skipped backup ${app.fqdn}`); + return iteratorCallback(null, null); // nothing to backup + } + + const startTime = new Date(); + backupAppWithTag(app, tag, options, (progress) => progressCallback({ percent: percent, message: progress.message }), function (error, backupId) { + if (error) { + debugApp(app, 'Unable to backup', error); + return iteratorCallback(error); + } + + debugApp(app, `Backed up. Took ${(new Date() - startTime)/1000} seconds`); + + iteratorCallback(null, backupId || null); // clear backupId if is in BAD_STATE and never backed up + }); + }, function appsBackedUp(error, backupIds) { + if (error) return callback(error); + + backupIds = backupIds.filter(function (id) { return id !== null; }); // remove apps in bad state that were never backed up + + progressCallback({ percent: percent, message: 'Backing up system data' }); + percent += step; + + backupBoxWithAppBackupIds(backupIds, tag, options, (progress) => progressCallback({ percent: percent, message: progress.message }), callback); + }); + }); +} diff --git a/src/mounts.js b/src/mounts.js index a1a8acb1a..f3b57989f 100644 --- a/src/mounts.js +++ b/src/mounts.js @@ -1,6 +1,7 @@ 'use strict'; exports = module.exports = { + isMountProvider, tryAddMount, removeMount, validateMountOptions, @@ -55,6 +56,10 @@ function validateMountOptions(type, options) { } } +function isMountProvider(provider) { + return provider === 'sshfs' || provider === 'cifs' || provider === 'nfs' || provider === 'ext4'; +} + // https://www.man7.org/linux/man-pages/man8/mount.8.html for various mount option flags // nfs - no_root_squash is mode on server to map all root to 'nobody' user. all_squash does this for all users (making it like ftp) // sshfs - supports users/permissions diff --git a/src/provision.js b/src/provision.js index bfba94668..d52be282b 100644 --- a/src/provision.js +++ b/src/provision.js @@ -10,6 +10,7 @@ exports = module.exports = { const assert = require('assert'), async = require('async'), backups = require('./backups.js'), + backuptask = require('./backuptask.js'), BoxError = require('./boxerror.js'), branding = require('./branding.js'), constants = require('./constants.js'), @@ -184,7 +185,7 @@ function restore(backupConfig, backupId, version, sysinfoConfig, options, auditS if (error) return done(error); if (activated) return done(new BoxError(BoxError.CONFLICT, 'Already activated. Restore with a fresh Cloudron installation.')); - if (backups.isMountProvider(backupConfig.provider)) { + if (mounts.isMountProvider(backupConfig.provider)) { error = mounts.validateMountOptions(backupConfig.provider, backupConfig.mountOptions); if (error) return done(error); @@ -218,7 +219,7 @@ function restore(backupConfig, backupId, version, sysinfoConfig, options, auditS async.series([ setProgress.bind(null, 'restore', 'Downloading backup'), - backups.restore.bind(null, backupConfig, backupId, (progress) => setProgress('restore', progress.message, NOOP_CALLBACK)), + backuptask.restore.bind(null, backupConfig, backupId, (progress) => setProgress('restore', progress.message, NOOP_CALLBACK)), settings.setSysinfoConfig.bind(null, sysinfoConfig), reverseProxy.restoreFallbackCertificates, (done) => { diff --git a/src/routes/apps.js b/src/routes/apps.js index 4a7a97444..8612c6b4e 100644 --- a/src/routes/apps.js +++ b/src/routes/apps.js @@ -412,19 +412,18 @@ function repair(req, res, next) { }); } -function restore(req, res, next) { +async function restore(req, res, next) { assert.strictEqual(typeof req.body, 'object'); assert.strictEqual(typeof req.resource, 'object'); - var data = req.body; + const data = req.body; if (!data.backupId || typeof data.backupId !== 'string') return next(new HttpError(400, 'backupId must be non-empty string')); - apps.restore(req.resource, data.backupId, auditSource.fromRequest(req), function (error, result) { - if (error) return next(BoxError.toHttpError(error)); + const [error, result] = await safe(apps.restore(req.resource, data.backupId, auditSource.fromRequest(req))); + if (error) return next(BoxError.toHttpError(error)); - next(new HttpSuccess(202, { taskId: result.taskId })); - }); + next(new HttpSuccess(202, { taskId: result.taskId })); } function importApp(req, res, next) { @@ -744,20 +743,19 @@ function execWebSocket(req, res, next) { }); } -function listBackups(req, res, next) { +async function listBackups(req, res, next) { assert.strictEqual(typeof req.resource, 'object'); - var page = typeof req.query.page !== 'undefined' ? parseInt(req.query.page) : 1; + const page = typeof req.query.page !== 'undefined' ? parseInt(req.query.page) : 1; if (!page || page < 0) return next(new HttpError(400, 'page query param has to be a postive number')); - var perPage = typeof req.query.per_page !== 'undefined'? parseInt(req.query.per_page) : 25; + const perPage = typeof req.query.per_page !== 'undefined'? parseInt(req.query.per_page) : 25; if (!perPage || perPage < 0) return next(new HttpError(400, 'per_page query param has to be a postive number')); - apps.listBackups(req.resource, page, perPage, function (error, result) { - if (error) return next(BoxError.toHttpError(error)); + const [error, result] = await safe(apps.listBackups(req.resource, page, perPage)); + if (error) return next(BoxError.toHttpError(error)); - next(new HttpSuccess(200, { backups: result })); - }); + next(new HttpSuccess(200, { backups: result })); } function uploadFile(req, res, next) { diff --git a/src/routes/backups.js b/src/routes/backups.js index edb795960..8b48f0e46 100644 --- a/src/routes/backups.js +++ b/src/routes/backups.js @@ -13,18 +13,17 @@ const auditSource = require('../auditsource.js'), HttpSuccess = require('connect-lastmile').HttpSuccess, safe = require('safetydance'); -function list(req, res, next) { - var page = typeof req.query.page !== 'undefined' ? parseInt(req.query.page) : 1; +async function list(req, res, next) { + const page = typeof req.query.page !== 'undefined' ? parseInt(req.query.page) : 1; if (!page || page < 0) return next(new HttpError(400, 'page query param has to be a postive number')); - var perPage = typeof req.query.per_page !== 'undefined'? parseInt(req.query.per_page) : 25; + const perPage = typeof req.query.per_page !== 'undefined'? parseInt(req.query.per_page) : 25; if (!perPage || perPage < 0) return next(new HttpError(400, 'per_page query param has to be a postive number')); - backups.getByIdentifierAndStatePaged(backups.BACKUP_IDENTIFIER_BOX, backups.BACKUP_STATE_NORMAL, page, perPage, function (error, result) { - if (error) return next(BoxError.toHttpError(error)); + const [error, result] = await safe(backups.getByIdentifierAndStatePaged(backups.BACKUP_IDENTIFIER_BOX, backups.BACKUP_STATE_NORMAL, page, perPage)); + if (error) return next(BoxError.toHttpError(error)); - next(new HttpSuccess(200, { backups: result })); - }); + next(new HttpSuccess(200, { backups: result })); } function startBackup(req, res, next) { diff --git a/src/scripts/backupupload.js b/src/scripts/backupupload.js index ae5c73beb..3b99d1a4c 100755 --- a/src/scripts/backupupload.js +++ b/src/scripts/backupupload.js @@ -4,9 +4,9 @@ if (process.argv[2] === '--check') return console.log('OK'); -var assert = require('assert'), +const assert = require('assert'), async = require('async'), - backups = require('../backups.js'), + backuptask = require('../backuptask.js'), database = require('../database.js'), debug = require('debug')('box:backupupload'), settings = require('../settings.js'), @@ -66,7 +66,7 @@ initialize(function (error) { dumpMemoryInfo(); const timerId = setInterval(dumpMemoryInfo, 30000); - backups.upload(backupId, format, dataLayoutString, throttledProgressCallback(5000), function resultHandler(error) { + backuptask.upload(backupId, format, dataLayoutString, throttledProgressCallback(5000), function resultHandler(error) { debug('upload completed. error: ', error); process.send({ result: error ? error.message : '' }); diff --git a/src/settings.js b/src/settings.js index 3c301d3bf..33967cf0f 100644 --- a/src/settings.js +++ b/src/settings.js @@ -417,14 +417,14 @@ function setBackupConfig(backupConfig, callback) { backups.injectPrivateFields(backupConfig, oldConfig); - if (backups.isMountProvider(backupConfig.provider) && (!backups.isMountProvider(oldConfig.provider) || mountOptionsChanged(oldConfig, backupConfig))) { + if (mounts.isMountProvider(backupConfig.provider) && (!mounts.isMountProvider(oldConfig.provider) || mountOptionsChanged(oldConfig, backupConfig))) { error = mounts.validateMountOptions(backupConfig.provider, backupConfig.mountOptions); if (error) return callback(error); [error] = await safe(mounts.tryAddMount(mountObject(backupConfig), { timeout: 10 })); // 10 seconds if (error) { - if (backups.isMountProvider(oldConfig.provider)) { // put back the old mount configuration + if (mounts.isMountProvider(oldConfig.provider)) { // put back the old mount configuration debug('setBackupConfig: rolling back to previous mount configuration'); await safe(mounts.tryAddMount(mountObject(oldConfig), { timeout: 10 })); @@ -451,7 +451,7 @@ function setBackupConfig(backupConfig, callback) { settingsdb.set(exports.BACKUP_CONFIG_KEY, JSON.stringify(backupConfig), async function (error) { if (error) return callback(error); - if (backups.isMountProvider(oldConfig.provider) && !backups.isMountProvider(backupConfig.provider)) { + if (mounts.isMountProvider(oldConfig.provider) && !mounts.isMountProvider(backupConfig.provider)) { debug('setBackupConfig: removing old backup mount point'); await safe(mounts.removeMount(mountObject(oldConfig))); } diff --git a/src/storage.js b/src/storage.js new file mode 100644 index 000000000..9b1b06903 --- /dev/null +++ b/src/storage.js @@ -0,0 +1,53 @@ +'use strict'; + +exports = module.exports = { + api, + + getBackupFilePath, +}; + +const assert = require('assert'), + path = require('path'); + +// choose which storage backend we use for test purpose we use s3 +function api(provider) { + switch (provider) { + case 'nfs': return require('./storage/filesystem.js'); + case 'cifs': return require('./storage/filesystem.js'); + case 'sshfs': return require('./storage/filesystem.js'); + case 'mountpoint': return require('./storage/filesystem.js'); + case 'ext4': return require('./storage/filesystem.js'); + case 's3': return require('./storage/s3.js'); + case 'gcs': return require('./storage/gcs.js'); + case 'filesystem': return require('./storage/filesystem.js'); + case 'minio': return require('./storage/s3.js'); + case 's3-v4-compat': return require('./storage/s3.js'); + case 'digitalocean-spaces': return require('./storage/s3.js'); + case 'exoscale-sos': return require('./storage/s3.js'); + case 'wasabi': return require('./storage/s3.js'); + case 'scaleway-objectstorage': return require('./storage/s3.js'); + case 'backblaze-b2': return require('./storage/s3.js'); + case 'linode-objectstorage': return require('./storage/s3.js'); + case 'ovh-objectstorage': return require('./storage/s3.js'); + case 'ionos-objectstorage': return require('./storage/s3.js'); + case 'vultr-objectstorage': return require('./storage/s3.js'); + case 'noop': return require('./storage/noop.js'); + default: return null; + } +} + +// This is not part of the storage api, since we don't want to pull the "format" logistics into that +function getBackupFilePath(backupConfig, backupId, format) { + assert.strictEqual(typeof backupConfig, 'object'); + assert.strictEqual(typeof backupId, 'string'); + assert.strictEqual(typeof format, 'string'); + + const backupPath = api(backupConfig.provider).getBackupPath(backupConfig); + + if (format === 'tgz') { + const fileType = backupConfig.encryption ? '.tar.gz.enc' : '.tar.gz'; + return path.join(backupPath, backupId+fileType); + } else { + return path.join(backupPath, backupId); + } +} diff --git a/src/taskworker.js b/src/taskworker.js index 11609d9ec..cf419578b 100755 --- a/src/taskworker.js +++ b/src/taskworker.js @@ -4,7 +4,8 @@ const apptask = require('./apptask.js'), async = require('async'), - backups = require('./backups.js'), + backupCleaner = require('./backupcleaner.js'), + backuptask = require('./backuptask.js'), cloudron = require('./cloudron.js'), database = require('./database.js'), domains = require('./domains.js'), @@ -19,11 +20,11 @@ const apptask = require('./apptask.js'), const TASKS = { // indexed by task type app: apptask.run, - backup: backups.backupBoxAndApps, + backup: backuptask.backupBoxAndApps, update: updater.update, checkCerts: reverseProxy.checkCerts, setupDnsAndCert: cloudron.setupDnsAndCert, - cleanBackups: backups.cleanup, + cleanBackups: backupCleaner.run, syncExternalLdap: externalLdap.sync, changeMailLocation: mail.changeLocation, syncDnsRecords: domains.syncDnsRecords, diff --git a/src/test/apps-test.js b/src/test/apps-test.js index 35ceacd16..ce9d4db10 100644 --- a/src/test/apps-test.js +++ b/src/test/apps-test.js @@ -10,12 +10,10 @@ const appdb = require('../appdb.js'), async = require('async'), constants = require('../constants.js'), BoxError = require('../boxerror.js'), - database = require('../database.js'), common = require('./common.js'), domains = require('../domains.js'), expect = require('expect.js'), hat = require('../hat.js'), - provision = require('../provision.js'), userdb = require('../userdb.js'); let AUDIT_SOURCE = { ip: '1.2.3.4' }; diff --git a/src/test/backups-test.js b/src/test/backups-test.js index f3f02cb00..5e0a8db85 100644 --- a/src/test/backups-test.js +++ b/src/test/backups-test.js @@ -88,6 +88,145 @@ describe('backups', function () { after(common.cleanup); + describe('backup', function () { + + it('add succeeds', function (done) { + var backup = { + id: 'backup-box', + encryptionVersion: 2, + packageVersion: '1.0.0', + type: backups.BACKUP_TYPE_BOX, + state: backups.BACKUP_STATE_NORMAL, + identifier: 'box', + dependsOn: [ 'dep1' ], + manifest: null, + format: 'tgz' + }; + + backupdb.add(backup.id, backup, function (error) { + expect(error).to.be(null); + done(); + }); + }); + + it('get succeeds', function (done) { + backupdb.get('backup-box', function (error, result) { + expect(error).to.be(null); + expect(result.encryptionVersion).to.be(2); + expect(result.packageVersion).to.be('1.0.0'); + expect(result.type).to.be(backups.BACKUP_TYPE_BOX); + expect(result.state).to.be(backups.BACKUP_STATE_NORMAL); + expect(result.creationTime).to.be.a(Date); + expect(result.dependsOn).to.eql(['dep1']); + expect(result.manifest).to.eql(null); + done(); + }); + }); + + it('get of unknown id fails', function (done) { + backupdb.get('somerandom', function (error, result) { + expect(error).to.be.a(BoxError); + expect(error.reason).to.be(BoxError.NOT_FOUND); + expect(result).to.not.be.ok(); + done(); + }); + }); + + it('getByTypePaged succeeds', function (done) { + backupdb.getByTypePaged(backups.BACKUP_TYPE_BOX, 1, 5, function (error, results) { + expect(error).to.be(null); + expect(results).to.be.an(Array); + expect(results.length).to.be(1); + + expect(results[0].id).to.be('backup-box'); + expect(results[0].encryptionVersion).to.be(2); + expect(results[0].packageVersion).to.be('1.0.0'); + expect(results[0].dependsOn).to.eql(['dep1']); + expect(results[0].manifest).to.eql(null); + + done(); + }); + }); + + it('delete succeeds', function (done) { + backupdb.del('backup-box', function (error, result) { + expect(error).to.be(null); + expect(result).to.not.be.ok(); + + backupdb.get('backup-box', function (error, result) { + expect(error).to.be.a(BoxError); + expect(error.reason).to.equal(BoxError.NOT_FOUND); + expect(result).to.not.be.ok(); + done(); + }); + }); + }); + + it('add app succeeds', function (done) { + var backup = { + id: 'app_appid_123', + encryptionVersion: null, + packageVersion: '1.0.0', + type: backups.BACKUP_TYPE_APP, + state: backups.BACKUP_STATE_CREATING, + identifier: 'appid', + dependsOn: [ ], + manifest: { foo: 'bar' }, + format: 'tgz' + }; + + backupdb.add(backup.id, backup, function (error) { + expect(error).to.be(null); + done(); + }); + }); + + it('get succeeds', function (done) { + backupdb.get('app_appid_123', function (error, result) { + expect(error).to.be(null); + expect(result.encryptionVersion).to.be(null); + expect(result.packageVersion).to.be('1.0.0'); + expect(result.type).to.be(backups.BACKUP_TYPE_APP); + expect(result.state).to.be(backups.BACKUP_STATE_CREATING); + expect(result.creationTime).to.be.a(Date); + expect(result.dependsOn).to.eql([]); + expect(result.manifest).to.eql({ foo: 'bar' }); + done(); + }); + }); + + it('getByIdentifierPaged succeeds', function (done) { + backupdb.getByIdentifierPaged('appid', 1, 5, function (error, results) { + expect(error).to.be(null); + expect(results).to.be.an(Array); + expect(results.length).to.be(1); + + expect(results[0].id).to.be('app_appid_123'); + expect(results[0].encryptionVersion).to.be(null); + expect(results[0].packageVersion).to.be('1.0.0'); + expect(results[0].dependsOn).to.eql([]); + expect(results[0].manifest).to.eql({ foo: 'bar' }); + + done(); + }); + }); + + it('delete succeeds', function (done) { + backupdb.del('app_appid_123', function (error, result) { + expect(error).to.be(null); + expect(result).to.not.be.ok(); + + backupdb.get('app_appid_123', function (error, result) { + expect(error).to.be.a(BoxError); + expect(error.reason).to.equal(BoxError.NOT_FOUND); + expect(result).to.not.be.ok(); + done(); + }); + }); + }); + + }); + describe('retention policy', function () { it('keeps latest', function () { let backup = { creationTime: moment().subtract(5, 's').toDate(), state: backups.BACKUP_STATE_NORMAL }; diff --git a/src/test/database-test.js b/src/test/database-test.js index 88a3d2c7b..629255237 100644 --- a/src/test/database-test.js +++ b/src/test/database-test.js @@ -8,8 +8,6 @@ const appdb = require('../appdb.js'), apps = require('../apps.js'), async = require('async'), - backupdb = require('../backupdb.js'), - backups = require('../backups.js'), BoxError = require('../boxerror.js'), constants = require('../constants.js'), database = require('../database'), @@ -883,145 +881,6 @@ describe('database', function () { }); - describe('backup', function () { - - it('add succeeds', function (done) { - var backup = { - id: 'backup-box', - encryptionVersion: 2, - packageVersion: '1.0.0', - type: backups.BACKUP_TYPE_BOX, - state: backups.BACKUP_STATE_NORMAL, - identifier: 'box', - dependsOn: [ 'dep1' ], - manifest: null, - format: 'tgz' - }; - - backupdb.add(backup.id, backup, function (error) { - expect(error).to.be(null); - done(); - }); - }); - - it('get succeeds', function (done) { - backupdb.get('backup-box', function (error, result) { - expect(error).to.be(null); - expect(result.encryptionVersion).to.be(2); - expect(result.packageVersion).to.be('1.0.0'); - expect(result.type).to.be(backups.BACKUP_TYPE_BOX); - expect(result.state).to.be(backups.BACKUP_STATE_NORMAL); - expect(result.creationTime).to.be.a(Date); - expect(result.dependsOn).to.eql(['dep1']); - expect(result.manifest).to.eql(null); - done(); - }); - }); - - it('get of unknown id fails', function (done) { - backupdb.get('somerandom', function (error, result) { - expect(error).to.be.a(BoxError); - expect(error.reason).to.be(BoxError.NOT_FOUND); - expect(result).to.not.be.ok(); - done(); - }); - }); - - it('getByTypePaged succeeds', function (done) { - backupdb.getByTypePaged(backups.BACKUP_TYPE_BOX, 1, 5, function (error, results) { - expect(error).to.be(null); - expect(results).to.be.an(Array); - expect(results.length).to.be(1); - - expect(results[0].id).to.be('backup-box'); - expect(results[0].encryptionVersion).to.be(2); - expect(results[0].packageVersion).to.be('1.0.0'); - expect(results[0].dependsOn).to.eql(['dep1']); - expect(results[0].manifest).to.eql(null); - - done(); - }); - }); - - it('delete succeeds', function (done) { - backupdb.del('backup-box', function (error, result) { - expect(error).to.be(null); - expect(result).to.not.be.ok(); - - backupdb.get('backup-box', function (error, result) { - expect(error).to.be.a(BoxError); - expect(error.reason).to.equal(BoxError.NOT_FOUND); - expect(result).to.not.be.ok(); - done(); - }); - }); - }); - - it('add app succeeds', function (done) { - var backup = { - id: 'app_appid_123', - encryptionVersion: null, - packageVersion: '1.0.0', - type: backups.BACKUP_TYPE_APP, - state: backups.BACKUP_STATE_CREATING, - identifier: 'appid', - dependsOn: [ ], - manifest: { foo: 'bar' }, - format: 'tgz' - }; - - backupdb.add(backup.id, backup, function (error) { - expect(error).to.be(null); - done(); - }); - }); - - it('get succeeds', function (done) { - backupdb.get('app_appid_123', function (error, result) { - expect(error).to.be(null); - expect(result.encryptionVersion).to.be(null); - expect(result.packageVersion).to.be('1.0.0'); - expect(result.type).to.be(backups.BACKUP_TYPE_APP); - expect(result.state).to.be(backups.BACKUP_STATE_CREATING); - expect(result.creationTime).to.be.a(Date); - expect(result.dependsOn).to.eql([]); - expect(result.manifest).to.eql({ foo: 'bar' }); - done(); - }); - }); - - it('getByIdentifierPaged succeeds', function (done) { - backupdb.getByIdentifierPaged('appid', 1, 5, function (error, results) { - expect(error).to.be(null); - expect(results).to.be.an(Array); - expect(results.length).to.be(1); - - expect(results[0].id).to.be('app_appid_123'); - expect(results[0].encryptionVersion).to.be(null); - expect(results[0].packageVersion).to.be('1.0.0'); - expect(results[0].dependsOn).to.eql([]); - expect(results[0].manifest).to.eql({ foo: 'bar' }); - - done(); - }); - }); - - it('delete succeeds', function (done) { - backupdb.del('app_appid_123', function (error, result) { - expect(error).to.be(null); - expect(result).to.not.be.ok(); - - backupdb.get('app_appid_123', function (error, result) { - expect(error).to.be.a(BoxError); - expect(error.reason).to.equal(BoxError.NOT_FOUND); - expect(result).to.not.be.ok(); - done(); - }); - }); - }); - - }); - describe('importFromFile', function () { before(function (done) { async.series([