diff --git a/src/apps.js b/src/apps.js index b6c3be73c..070fa5575 100644 --- a/src/apps.js +++ b/src/apps.js @@ -97,6 +97,8 @@ exports = module.exports = { writeConfig, loadConfig, + canBackupApp, + PORT_TYPE_TCP: 'tcp', PORT_TYPE_UDP: 'udp', @@ -114,7 +116,6 @@ exports = module.exports = { ISTATE_PENDING_RESTORE: 'pending_restore', // restore to previous backup or on upgrade ISTATE_PENDING_IMPORT: 'pending_import', // import from external backup ISTATE_PENDING_UPDATE: 'pending_update', // update from installed state preserving data - ISTATE_PENDING_BACKUP: 'pending_backup', // backup the app. this is state because it blocks other operations ISTATE_PENDING_START: 'pending_start', ISTATE_PENDING_STOP: 'pending_stop', ISTATE_PENDING_RESTART: 'pending_restart', @@ -1247,11 +1248,6 @@ async function onTaskFinished(error, appId, installationState, taskId, auditSour await notifications.unpin(notifications.TYPE_MANUAL_APP_UPDATE_NEEDED, { context: app.id }); break; } - case exports.ISTATE_PENDING_BACKUP: { - const backup = task.result ? await backups.get(task.result) : null; // if task crashed, no result - await eventlog.add(eventlog.ACTION_APP_BACKUP_FINISH, auditSource, { app, success, errorMessage, remotePath: backup?.remotePath, backupId: task.result }); - break; - } } } @@ -1264,7 +1260,7 @@ async function scheduleTask(appId, installationState, taskId, auditSource) { const backupConfig = await backups.getConfig(); let memoryLimit = 400; - if (installationState === exports.ISTATE_PENDING_BACKUP || installationState === exports.ISTATE_PENDING_CLONE || installationState === exports.ISTATE_PENDING_RESTORE + if (installationState === exports.ISTATE_PENDING_CLONE || installationState === exports.ISTATE_PENDING_RESTORE || installationState === exports.ISTATE_PENDING_IMPORT || installationState === exports.ISTATE_PENDING_UPDATE) { memoryLimit = backupConfig.limits?.memoryLimit ? Math.max(backupConfig.limits.memoryLimit/1024/1024, 400) : 400; } else if (installationState === exports.ISTATE_PENDING_DATA_DIR_MIGRATION) { @@ -1331,7 +1327,7 @@ function checkAppState(app, state) { if (app.runState === exports.RSTATE_STOPPED) { // can't backup or restore since app addons are down. can't update because migration scripts won't run - if (state === exports.ISTATE_PENDING_UPDATE || state === exports.ISTATE_PENDING_BACKUP || state === exports.ISTATE_PENDING_RESTORE || state === exports.ISTATE_PENDING_IMPORT) return new BoxError(BoxError.BAD_STATE, 'Not allowed in stopped state'); + if (state === exports.ISTATE_PENDING_UPDATE || state === exports.ISTATE_PENDING_RESTORE || state === exports.ISTATE_PENDING_IMPORT) return new BoxError(BoxError.BAD_STATE, 'Not allowed in stopped state'); } return null; @@ -2404,15 +2400,10 @@ async function exportApp(app, data, auditSource) { const appId = app.id; - const error = checkAppState(app, exports.ISTATE_PENDING_BACKUP); - if (error) throw error; + if (!canBackupApp(app)) throw new BoxError(BoxError.BAD_STATE, 'App cannot be backed up in this state'); - const task = { - args: { snapshotOnly: true }, - values: {} - }; - - const taskId = await addTask(appId, exports.ISTATE_PENDING_BACKUP, task, auditSource); + const taskId = await tasks.add(`${tasks.TASK_APP_BACKUP_PREFIX}${app.id}`, [ appId, { snapshotOnly: true } ]); + safe(tasks.startTask(taskId, {}), { debug }); // background return { taskId }; } @@ -2766,21 +2757,41 @@ async function getExec(app, execId) { return await docker.getExec(execId); } +function canBackupApp(app) { + // only backup apps that are installed or specific pending states + + // stopped apps cannot be backed up because addons might be down (redis) + if (app.runState === exports.RSTATE_STOPPED) return false; + + // we used to check the health here but that doesn't work for stopped apps. it's better to just fail + // and inform the user if the backup fails and the app addons have not been setup yet. + return app.installationState === exports.ISTATE_INSTALLED || + app.installationState === exports.ISTATE_PENDING_CONFIGURE || + app.installationState === exports.ISTATE_PENDING_UPDATE; // called from apptask +} + async function backup(app, auditSource) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof auditSource, 'object'); - const appId = app.id; + if (!canBackupApp(app)) throw new BoxError(BoxError.BAD_STATE, 'App cannot be backed up in this state'); - const error = checkAppState(app, exports.ISTATE_PENDING_BACKUP); - if (error) throw error; + const taskId = await tasks.add(`${tasks.TASK_APP_BACKUP_PREFIX}${app.id}`, [ app.id, { snapshotOnly: false } ]); - const task = { - args: {}, - values: {} - }; - const taskId = await addTask(appId, exports.ISTATE_PENDING_BACKUP, task, auditSource); - await eventlog.add(eventlog.ACTION_APP_BACKUP, auditSource, { app, appId, taskId }); + const backupConfig = await backups.getConfig(); + const memoryLimit = backupConfig.limits?.memoryLimit ? Math.max(backupConfig.limits.memoryLimit/1024/1024, 1024) : 1024; + + // background + tasks.startTask(taskId, { timeout: 24 * 60 * 60 * 1000 /* 24 hours */, nice: 15, memoryLimit, oomScoreAdjust: -999 }) + .then(async (backupId) => { + const backup = await backups.get(backupId); // if task crashed, no result + await eventlog.add(eventlog.ACTION_APP_BACKUP_FINISH, auditSource, { app, success: !!backup, errorMessage: '', remotePath: backup?.remotePath, backupId: backupId }); + }) + .catch(async (error) => { + await eventlog.add(eventlog.ACTION_APP_BACKUP_FINISH, auditSource, { app, success: false, errorMessage: error.message }); + }); + + await eventlog.add(eventlog.ACTION_APP_BACKUP, auditSource, { app, appId: app.id, taskId }); return { taskId }; } diff --git a/src/apptask.js b/src/apptask.js index d967f6eb5..3538e3d4f 100644 --- a/src/apptask.js +++ b/src/apptask.js @@ -371,21 +371,6 @@ async function installCommand(app, args, progressCallback) { await updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null }); } -async function backupCommand(app, args, progressCallback) { - assert.strictEqual(typeof app, 'object'); - assert.strictEqual(typeof args, 'object'); - assert.strictEqual(typeof progressCallback, 'function'); - - await progressCallback({ percent: 10, message: 'Backing up' }); - const backupId = await backuptask.backupApp(app, { snapshotOnly: !!args.snapshotOnly }, (progress) => { - progressCallback({ percent: 30, message: progress.message }); - }); - - await progressCallback({ percent: 100, message: 'Done' }); - await updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: null }); - return backupId; -} - // this command can also be called when the app is stopped. do not touch services async function recreateCommand(app, args, progressCallback) { assert.strictEqual(typeof app, 'object'); @@ -796,9 +781,6 @@ async function run(appId, args, progressCallback) { case apps.ISTATE_PENDING_UPDATE: cmd = updateCommand(app, args, progressCallback); break; - case apps.ISTATE_PENDING_BACKUP: - cmd = backupCommand(app, args, progressCallback); - break; case apps.ISTATE_PENDING_START: cmd = startCommand(app, args, progressCallback); break; @@ -820,10 +802,7 @@ async function run(appId, args, progressCallback) { if (error) { debug(`run: app error for state ${app.installationState}: %o`, error); - if (app.installationState === apps.ISTATE_PENDING_BACKUP) { - // return to installed state intentionally. the error is stashed only in the task and not the app (the UI shows error state otherwise) - await safe(updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: null }), { debug }); - } else if (app.installationState === apps.ISTATE_PENDING_UPDATE && error.backupError) { + if (app.installationState === apps.ISTATE_PENDING_UPDATE && error.backupError) { debug('run: update aborted because backup failed'); await safe(updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null }, { debug })); } else { diff --git a/src/backuptask.js b/src/backuptask.js index 9c39a325e..c70eeaa47 100644 --- a/src/backuptask.js +++ b/src/backuptask.js @@ -2,10 +2,10 @@ exports = module.exports = { fullBackup, + appBackup, restore, - backupApp, downloadApp, backupMail, @@ -34,20 +34,6 @@ const apps = require('./apps.js'), const BACKUP_UPLOAD_CMD = path.join(__dirname, 'scripts/backupupload.js'); -function canBackupApp(app) { - // only backup apps that are installed or specific pending states - - // stopped apps cannot be backed up because addons might be down (redis) - if (app.runState === apps.RSTATE_STOPPED) return false; - - // we used to check the health here but that doesn't work for stopped apps. it's better to just fail - // and inform the user if the backup fails and the app addons have not been setup yet. - return app.installationState === apps.ISTATE_INSTALLED || - app.installationState === apps.ISTATE_PENDING_CONFIGURE || - app.installationState === apps.ISTATE_PENDING_BACKUP || // called from apptask - app.installationState === apps.ISTATE_PENDING_UPDATE; // called from apptask -} - async function checkPreconditions(backupConfig, dataLayout) { assert.strictEqual(typeof backupConfig, 'object'); assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout'); @@ -316,20 +302,6 @@ async function rotateAppBackup(backupConfig, app, tag, options, progressCallback return id; } -async function backupApp(app, options, progressCallback) { - assert.strictEqual(typeof app, 'object'); - assert.strictEqual(typeof options, 'object'); - assert.strictEqual(typeof progressCallback, 'function'); - - if (options.snapshotOnly) return await snapshotApp(app, progressCallback); - - const tag = (new Date()).toISOString().replace(/[T.]/g, '-').replace(/[:Z]/g,''); - - debug(`backupApp: backing up ${app.fqdn} with tag ${tag}`); - - return await backupAppWithTag(app, tag, options, progressCallback); -} - async function snapshotApp(app, progressCallback) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof progressCallback, 'function'); @@ -380,7 +352,7 @@ async function backupAppWithTag(app, tag, options, progressCallback) { assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof progressCallback, 'function'); - if (!canBackupApp(app)) { // if we cannot backup, reuse it's most recent backup + if (!apps.canBackupApp(app)) { // if we cannot backup, reuse it's most recent backup const results = await backups.getByIdentifierAndStatePaged(app.id, backups.BACKUP_STATE_NORMAL, 1, 1); if (results.length === 0) return null; // no backup to re-use @@ -510,11 +482,11 @@ async function fullBackup(options, progressCallback) { } progressCallback({ percent, message: `Backing up ${app.fqdn} (${i+1}/${allApps.length}). Waiting for lock` }); - await locks.wait(`${locks.TYPE_APP_TASK_PREFIX}${app.id}`); + await locks.wait(`${locks.TYPE_APP_BACKUP_PREFIX}${app.id}`); const startTime = new Date(); const [appBackupError, appBackupId] = await safe(backupAppWithTag(app, tag, options, (progress) => progressCallback({ percent, message: progress.message }))); debug(`fullBackup: app ${app.fqdn} backup finished. Took ${(new Date() - startTime)/1000} seconds`); - await locks.release(`${locks.TYPE_APP_TASK_PREFIX}${app.id}`); + await locks.release(`${locks.TYPE_APP_BACKUP_PREFIX}${app.id}`); if (appBackupError) throw appBackupError; if (appBackupId) appBackupIds.push(appBackupId); // backupId can be null if in BAD_STATE and never backed up } @@ -530,3 +502,28 @@ async function fullBackup(options, progressCallback) { const backupId = await backupBox(dependsOn, tag, options, (progress) => progressCallback({ percent, message: progress.message })); return backupId; } + +// this function is called from external process +async function appBackup(appId, options, progressCallback) { + assert.strictEqual(typeof appId, 'string'); + assert.strictEqual(typeof options, 'object'); + assert.strictEqual(typeof progressCallback, 'function'); + + const app = await apps.get(appId); + if (!app) throw new BoxError(BoxError.BAD_FIELD, 'App not found'); + + let backupId = null; + await progressCallback({ percent: 1, message: `Backing up ${app.fqdn}. Waiting for lock` }); + await locks.wait(`${locks.TYPE_APP_BACKUP_PREFIX}${app.id}`); + const startTime = new Date(); + if (options.snapshotOnly) { + await snapshotApp(app, progressCallback); + } else { + const tag = (new Date()).toISOString().replace(/[T.]/g, '-').replace(/[:Z]/g,''); + backupId = await backupAppWithTag(app, tag, options, progressCallback); + } + await locks.release(`${locks.TYPE_APP_BACKUP_PREFIX}${app.id}`); + + await progressCallback({ percent: 100, message: `app ${app.fqdn} backup finished. Took ${(new Date() - startTime)/1000} seconds` }); + return backupId; +} diff --git a/src/locks.js b/src/locks.js index 16588d4e0..cc7792084 100644 --- a/src/locks.js +++ b/src/locks.js @@ -11,6 +11,7 @@ exports = module.exports = { releaseByTaskId, TYPE_APP_TASK_PREFIX: 'app_task_', + TYPE_APP_BACKUP_PREFIX: 'app_backup_', TYPE_BOX_UPDATE: 'box_update', // for the actual update and after the backup. this allows the backup before update do not block TYPE_BOX_UPDATE_TASK: 'box_update_task', // for scheduling the update task TYPE_FULL_BACKUP_TASK: 'full_backup_task', // for scheduling the backup task @@ -50,7 +51,8 @@ function canAcquire(data, type) { assert.strictEqual(typeof type, 'string'); if (type === exports.TYPE_BOX_UPDATE) { - if (Object.keys(data).some(k => k.startsWith(exports.TYPE_APP_TASK_PREFIX))) return new BoxError(BoxError.BAD_STATE, 'One or more apptasks are active'); + if (Object.keys(data).some(k => k.startsWith(exports.TYPE_APP_TASK_PREFIX))) return new BoxError(BoxError.BAD_STATE, 'One or more app tasks are active'); + if (Object.keys(data).some(k => k.startsWith(exports.TYPE_APP_BACKUP_PREFIX))) return new BoxError(BoxError.BAD_STATE, 'One or more app backups are active'); } else if (type.startsWith(exports.TYPE_APP_TASK_PREFIX)) { if (exports.TYPE_BOX_UPDATE in data) return new BoxError(BoxError.BAD_STATE, 'Update is active'); } else if (type === exports.TYPE_FULL_BACKUP_TASK) { @@ -59,6 +61,8 @@ function canAcquire(data, type) { if (exports.TYPE_FULL_BACKUP_TASK in data) return new BoxError(BoxError.BAD_STATE, 'Backup task is active'); } + // TYPE_MAIL_SERVER_RESTART can co-run with everything + return null; } diff --git a/src/tasks.js b/src/tasks.js index 854cd0116..be70f8f02 100644 --- a/src/tasks.js +++ b/src/tasks.js @@ -20,7 +20,8 @@ exports = module.exports = { // task types. if you add a task here, fill up the function table in taskworker and dashboard constants.js TASK_APP: 'app', - TASK_BACKUP: 'backup', + TASK_APP_BACKUP_PREFIX: 'appBackup_', + TASK_BACKUP: 'backup', // full backup TASK_BOX_UPDATE: 'boxUpdate', TASK_CHECK_CERTS: 'checkCerts', TASK_SYNC_DYNDNS: 'syncDyndns', diff --git a/src/taskworker.js b/src/taskworker.js index cafc1cf8c..5d528db63 100755 --- a/src/taskworker.js +++ b/src/taskworker.js @@ -22,6 +22,7 @@ const apptask = require('./apptask.js'), const TASKS = { // indexed by task type app: apptask.run, + appBackup: backuptask.appBackup, backup: backuptask.fullBackup, boxUpdate: updater.updateBox, checkCerts: reverseProxy.checkCerts,