diff --git a/src/backups.js b/src/backups.js index 67a6185fd..0a8f5ea81 100644 --- a/src/backups.js +++ b/src/backups.js @@ -12,7 +12,8 @@ exports = module.exports = { ensureBackup: ensureBackup, - backup: backup, + runBackupTask: runBackupTask, + restore: restore, backupApp: backupApp, @@ -40,6 +41,7 @@ var addons = require('./addons.js'), async = require('async'), assert = require('assert'), backupdb = require('./backupdb.js'), + child_process = require('child_process'), config = require('./config.js'), crypto = require('crypto'), database = require('./database.js'), @@ -182,7 +184,7 @@ function getBackupFilePath(backupConfig, backupId, format) { } function log(detail) { - safe.fs.appendFileSync(paths.BACKUP_LOG_FILE, detail + '\n', 'utf8'); + debug(detail); progress.setDetail(progress.BACKUP, detail); } @@ -388,7 +390,7 @@ function saveFsMetadata(appDataDir, callback) { callback(); } -// this function is called via backuptask (since it needs root to traverse app's directory) +// this function is called via backupupload (since it needs root to traverse app's directory) function upload(backupId, format, dataDir, callback) { assert.strictEqual(typeof backupId, 'string'); assert.strictEqual(typeof format, 'string'); @@ -538,8 +540,6 @@ function download(backupConfig, backupId, format, dataDir, callback) { assert.strictEqual(typeof dataDir, 'string'); assert.strictEqual(typeof callback, 'function'); - safe.fs.unlinkSync(paths.BACKUP_LOG_FILE); // start fresh log file - log(`Downloading ${backupId} of format ${format} to ${dataDir}`); if (format === 'tgz') { @@ -887,7 +887,6 @@ function backupApp(app, callback) { assert.strictEqual(typeof callback, 'function'); const timestamp = (new Date()).toISOString().replace(/[T.]/g, '-').replace(/[:Z]/g,''); - safe.fs.unlinkSync(paths.BACKUP_LOG_FILE); // start fresh log file progress.set(progress.BACKUP, 10, 'Backing up ' + app.fqdn); @@ -905,7 +904,6 @@ function backupBoxAndApps(auditSource, callback) { callback = callback || NOOP_CALLBACK; var timestamp = (new Date()).toISOString().replace(/[T.]/g, '-').replace(/[:Z]/g,''); - safe.fs.unlinkSync(paths.BACKUP_LOG_FILE); // start fresh log file eventlog.add(eventlog.ACTION_BACKUP_START, auditSource, { }); @@ -956,24 +954,42 @@ function backupBoxAndApps(auditSource, callback) { }); } -function backup(auditSource, callback) { +function runBackupTask(auditSource, callback) { assert.strictEqual(typeof auditSource, 'object'); assert.strictEqual(typeof callback, 'function'); - var error = locker.lock(locker.OP_FULL_BACKUP); + // fork and run! + let error = locker.lock(locker.OP_FULL_BACKUP); if (error) return callback(new BackupsError(BackupsError.BAD_STATE, error.message)); - var startTime = new Date(); + let startTime = new Date(); progress.set(progress.BACKUP, 0, 'Starting'); // ensure tools can 'wait' on progress - backupBoxAndApps(auditSource, function (error) { // start the backup operation in the background - if (error) { - debug('backup failed.', error); - mailer.backupFailed(error); + let fd = safe.fs.openSync(paths.BACKUP_LOG_FILE, 'a'); // will autoclose + if (!fd) { + debug('Unable to get log filedescriptor %s', safe.error.message); + return callback(safe.error); + } + + debug(`starting backuptask. logs at ${paths.BACKUP_LOG_FILE}`); + + // when parent process dies, this process is killed because KillMode=control-group in systemd unit file + let cp = child_process.fork(__dirname + '/backuptask.js', [ JSON.stringify(auditSource) ], { stdio: [ 'pipe', fd, fd, 'ipc' ]}); + cp.once('exit', function (code, signal) { + debug(`backuptask completed with code ${code} and signal ${signal}`); + + let error; + if (code === null /* signal */ || (code !== 0 && code !== 50)) { // apptask crashed + error = new Error(`backuptask completed with code ${code} and signal ${signal}`); + } else if (code === 50) { + error = new Error(safe.fs.readFileSync(paths.BACKUP_RESULT_FILE, 'utf8') || safe.error.message); } locker.unlock(locker.OP_FULL_BACKUP); + progress.set(progress.BACKUP, 100, error ? error.message : ''); + if (error) mailer.backupFailed(error); + debug('backup took %s seconds', (new Date() - startTime)/1000); }); @@ -999,7 +1015,7 @@ function ensureBackup(auditSource, callback) { return callback(null); } - backup(auditSource, callback); + runBackupTask(auditSource, callback); }); }); } diff --git a/src/backuptask.js b/src/backuptask.js new file mode 100755 index 000000000..753a39008 --- /dev/null +++ b/src/backuptask.js @@ -0,0 +1,46 @@ +#!/bin/bash +':' //# comment; exec /usr/bin/env node --max_old_space_size=300 "$0" "$@" + +// to understand the above hack read http://sambal.org/2014/02/passing-options-node-shebang-line/ + +'use strict'; + +require('supererror')({ splatchError: true }); + +var assert = require('assert'), + backups = require('./backups.js'), + database = require('./database.js'), + debug = require('debug')('box:backupupload'), + paths = require('./paths.js'), + safe = require('safetydance'); + +function initialize(callback) { + assert.strictEqual(typeof callback, 'function'); + + database.initialize(callback); +} + +// Main process starts here +const auditSource = JSON.parse(process.argv[2]); + +debug('Staring complete backup'); + +process.on('SIGTERM', function () { + process.exit(0); +}); + +initialize(function (error) { + if (error) throw error; + + safe.fs.writeFileSync(paths.BACKUP_RESULT_FILE, ''); + + backups.backupBoxAndApps(auditSource, function (error) { + if (error) debug('backup failed.', error); + + safe.fs.writeFileSync(paths.BACKUP_RESULT_FILE, error ? error.message : ''); + + // https://nodejs.org/api/process.html are exit codes used by node. apps.js uses the value below + // to check apptask crashes + process.exit(error ? 50 : 0); + }); +}); diff --git a/src/routes/backups.js b/src/routes/backups.js index 1135a008b..e3fe46abd 100644 --- a/src/routes/backups.js +++ b/src/routes/backups.js @@ -34,7 +34,7 @@ function get(req, res, next) { function create(req, res, next) { // note that cloudron.backup only waits for backup initiation and not for backup to complete // backup progress can be checked up ny polling the progress api call - backups.backup(auditSource(req), function (error) { + backups.runBackupTask(auditSource(req), function (error) { if (error && error.reason === BackupsError.BAD_STATE) return next(new HttpError(409, error.message)); if (error) return next(new HttpError(500, error)); diff --git a/src/routes/sysadmin.js b/src/routes/sysadmin.js index a0fd3b1f2..3ebc0250e 100644 --- a/src/routes/sysadmin.js +++ b/src/routes/sysadmin.js @@ -26,7 +26,7 @@ function backup(req, res, next) { // note that cloudron.backup only waits for backup initiation and not for backup to complete // backup progress can be checked up ny polling the progress api call var auditSource = { userId: null, username: 'sysadmin' }; - backups.backup(auditSource, function (error) { + backups.runBackupTask(auditSource, function (error) { if (error && error.reason === BackupsError.BAD_STATE) return next(new HttpError(409, error.message)); if (error) return next(new HttpError(500, error)); diff --git a/src/test/backups-test.js b/src/test/backups-test.js index 2ebe08f3e..21d3ed160 100644 --- a/src/test/backups-test.js +++ b/src/test/backups-test.js @@ -69,7 +69,7 @@ function compareDirectories(one, two, callback) { } function createBackup(callback) { - backups.backup({ username: 'test' }, function (error) { // this call does not wait for the backup! + backups.runBackupTask({ username: 'test' }, function (error) { // this call does not wait for the backup! if (error) return callback(error); function waitForBackup() {