diff --git a/setup/start.sh b/setup/start.sh index 9777177f7..ce2dc2366 100755 --- a/setup/start.sh +++ b/setup/start.sh @@ -84,7 +84,7 @@ mkdir -p "${PLATFORM_DATA_DIR}/addons/mail" mkdir -p "${PLATFORM_DATA_DIR}/collectd/collectd.conf.d" mkdir -p "${PLATFORM_DATA_DIR}/logrotate.d" mkdir -p "${PLATFORM_DATA_DIR}/acme" -mkdir -p "${PLATFORM_DATA_DIR}/snapshots" +mkdir -p "${PLATFORM_DATA_DIR}/backup" mkdir -p "${BOX_DATA_DIR}/appicons" mkdir -p "${BOX_DATA_DIR}/certs" @@ -269,7 +269,7 @@ CONF_END echo "==> Changing ownership" chown "${USER}:${USER}" -R "${CONFIG_DIR}" -chown "${USER}:${USER}" -R "${PLATFORM_DATA_DIR}/nginx" "${PLATFORM_DATA_DIR}/collectd" "${PLATFORM_DATA_DIR}/logrotate.d" "${PLATFORM_DATA_DIR}/addons" "${PLATFORM_DATA_DIR}/acme" "${PLATFORM_DATA_DIR}/snapshots" +chown "${USER}:${USER}" -R "${PLATFORM_DATA_DIR}/nginx" "${PLATFORM_DATA_DIR}/collectd" "${PLATFORM_DATA_DIR}/logrotate.d" "${PLATFORM_DATA_DIR}/addons" "${PLATFORM_DATA_DIR}/acme" "${PLATFORM_DATA_DIR}/backup" chown "${USER}:${USER}" -R "${BOX_DATA_DIR}" chown "${USER}:${USER}" -R "${BOX_DATA_DIR}/mail/dkim" # this is owned by box currently since it generates the keys chown "${USER}:${USER}" "${PLATFORM_DATA_DIR}/INFRA_VERSION" 2>/dev/null || true diff --git a/src/backups.js b/src/backups.js index 0265c361b..ff94b532d 100644 --- a/src/backups.js +++ b/src/backups.js @@ -219,10 +219,12 @@ function sync(backupConfig, backupId, dataDir, callback) { syncer.sync(dataDir, function processTask(task, iteratorCallback) { debug('processing task: %j', task); if (task.operation === 'add') { + safe.fs.writeFileSync(paths.BACKUP_RESULT_FILE, 'Adding ' + task.path); var stream = fs.createReadStream(path.join(dataDir, task.path)); stream.on('error', function () { return iteratorCallback(); }); // ignore error if file disappears api(backupConfig.provider).upload(backupConfig, getBackupFilePath(backupConfig, backupId, task.path), stream, iteratorCallback); } else if (task.operation === 'remove') { + safe.fs.writeFileSync(paths.BACKUP_RESULT_FILE, 'Removing ' + task.path); api(backupConfig.provider).remove(backupConfig, getBackupFilePath(backupConfig, backupId, task.path), iteratorCallback); } }, 10 /* concurrency */, function (error) { @@ -316,10 +318,10 @@ function createEmptyDirs(appDataDir, callback) { assert.strictEqual(typeof appDataDir, 'string'); assert.strictEqual(typeof callback, 'function'); - debugApp('createEmptyDirs: recreating empty directories'); + debug('createEmptyDirs: recreating empty directories'); var emptyDirs = safe.fs.readFileSync(path.join(appDataDir, 'emptydirs.txt'), 'utf8'); - if (!emptyDirs) return callback(new Error('emptydirs.txt was not found:' + safe.fs.error)); + if (emptyDirs === null) return callback(new Error('emptydirs.txt was not found:' + safe.error.message)); async.eachSeries(emptyDirs.trim().split('\n'), function createPath(emptyDir, iteratorDone) { mkdirp(path.join(appDataDir, 'data', emptyDir), iteratorDone); @@ -331,7 +333,7 @@ function download(backupId, dataDir, callback) { assert.strictEqual(typeof dataDir, 'string'); assert.strictEqual(typeof callback, 'function'); - debug('Start download of id %s', backupId); + debug('Start download of id %s to %s', backupId, dataDir); settings.getBackupConfig(function (error, backupConfig) { if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error)); @@ -356,10 +358,12 @@ function runBackupTask(backupId, dataDir, callback) { assert.strictEqual(typeof dataDir, 'string'); assert.strictEqual(typeof callback, 'function'); - var killTimerId = null; + var killTimerId = null, progressTimerId = null; - var cp = shell.sudo(`backup-${backupId}`, [ BACKUPTASK_CMD, backupId, dataDir ], { env: process.env }, function (error) { + var cp = shell.sudo(`backup-${backupId}`, [ BACKUPTASK_CMD, backupId, dataDir ], { env: process.env, logFile: paths.BACKUP_LOG_FILE }, function (error) { clearTimeout(killTimerId); + clearInterval(progressTimerId); + cp = null; if (error && (error.code === null /* signal */ || (error.code !== 0 && error.code !== 50))) { // backuptask crashed @@ -372,6 +376,11 @@ function runBackupTask(backupId, dataDir, callback) { callback(); }); + progressTimerId = setInterval(function () { + var result = safe.fs.readFileSync(paths.BACKUP_RESULT_FILE, 'utf8'); + if (result) progress.setDetail(progress.BACKUP, result); + }, 1000); // every second + killTimerId = setTimeout(function () { debug('runBackupTask: backup task taking too long. killing'); cp.kill(); @@ -848,6 +857,9 @@ function cleanupSnapshots(backupConfig, callback) { api(backupConfig.provider).remove(backupConfig, getBackupFilePath(backupConfig, `snapshot/app_${appId}`), function (/* ignoredError */) { setSnapshotInfo(appId, null); + safe.fs.unlinkSync(path.join(paths.BACKUP_INFO_DIR, `${appId}.sync.cache`)); + safe.fs.unlinkSync(path.join(paths.BACKUP_INFO_DIR, `${appId}.sync.cache.new`)); + iteratorDone(); }); }); diff --git a/src/backuptask.js b/src/backuptask.js index 1f8cd90d5..7c9fabe40 100755 --- a/src/backuptask.js +++ b/src/backuptask.js @@ -40,6 +40,8 @@ process.on('SIGTERM', function () { initialize(function (error) { if (error) throw error; + safe.fs.writeFileSync(paths.BACKUP_RESULT_FILE, ''); + backups.upload(backupId, dataDir, function resultHandler(error) { if (error) debug('completed with error', error); diff --git a/src/paths.js b/src/paths.js index 21f2b8eee..6dd863b21 100644 --- a/src/paths.js +++ b/src/paths.js @@ -7,7 +7,8 @@ var config = require('./config.js'), exports = module.exports = { CLOUDRON_DEFAULT_AVATAR_FILE: path.join(__dirname + '/../assets/avatar.png'), INFRA_VERSION_FILE: path.join(config.baseDir(), 'platformdata/INFRA_VERSION'), - BACKUP_RESULT_FILE: path.join(config.baseDir(), 'platformdata/backupresult'), + BACKUP_RESULT_FILE: path.join(config.baseDir(), 'platformdata/backup/result.txt'), + BACKUP_LOG_FILE: path.join(config.baseDir(), 'platformdata/backup/logs.txt'), OLD_DATA_DIR: path.join(config.baseDir(), 'data'), PLATFORM_DATA_DIR: path.join(config.baseDir(), 'platformdata'), @@ -21,8 +22,8 @@ exports = module.exports = { NGINX_CONFIG_DIR: path.join(config.baseDir(), 'platformdata/nginx'), NGINX_APPCONFIG_DIR: path.join(config.baseDir(), 'platformdata/nginx/applications'), NGINX_CERT_DIR: path.join(config.baseDir(), 'platformdata/nginx/cert'), - SNAPSHOT_DIR: path.join(config.baseDir(), 'platformdata/snapshots'), - SNAPSHOT_INFO_FILE: path.join(config.baseDir(), 'platformdata/snapshots/info.json'), + BACKUP_INFO_DIR: path.join(config.baseDir(), 'platformdata/backup'), + SNAPSHOT_INFO_FILE: path.join(config.baseDir(), 'platformdata/backup/snapshot-info.json'), // this is not part of appdata because an icon may be set before install APP_ICONS_DIR: path.join(config.baseDir(), 'boxdata/appicons'), diff --git a/src/progress.js b/src/progress.js index ef2b0b94d..bb445d978 100644 --- a/src/progress.js +++ b/src/progress.js @@ -2,6 +2,7 @@ exports = module.exports = { set: set, + setDetail: setDetail, clear: clear, get: get, @@ -29,12 +30,20 @@ function set(tag, percent, message) { progress[tag] = { percent: percent, - message: message + message: message, + detail: '' }; debug('%s: %s %s', tag, percent, message); } +function setDetail(tag, detail) { + assert.strictEqual(typeof tag, 'string'); + assert.strictEqual(typeof detail, 'string'); + + progress[tag].detail = detail; +} + function clear(tag) { assert.strictEqual(typeof tag, 'string'); diff --git a/src/shell.js b/src/shell.js index 1b5024b4b..30b8280bc 100644 --- a/src/shell.js +++ b/src/shell.js @@ -10,6 +10,7 @@ exports = module.exports = { var assert = require('assert'), child_process = require('child_process'), debug = require('debug')('box:shell'), + fs = require('fs'), once = require('once'), util = require('util'); @@ -41,15 +42,19 @@ function exec(tag, file, args, options, callback) { debug(tag + ' execFile: %s', file); // do not dump args as it might have sensitive info var cp = child_process.spawn(file, args, options); - if (!options.noDebugStdout) { + if (options.logFile) { + var logFile = fs.createWriteStream(options.logFile); + cp.stdout.pipe(logFile); + cp.stderr.pipe(logFile); + } else { cp.stdout.on('data', function (data) { debug(tag + ' (stdout): %s', data.toString('utf8')); }); - } - cp.stderr.on('data', function (data) { - debug(tag + ' (stderr): %s', data.toString('utf8')); - }); + cp.stderr.on('data', function (data) { + debug(tag + ' (stderr): %s', data.toString('utf8')); + }); + } cp.on('exit', function (code, signal) { if (code || signal) debug(tag + ' code: %s, signal: %s', code, signal); diff --git a/src/syncer.js b/src/syncer.js index 63d199b3e..c5678424b 100644 --- a/src/syncer.js +++ b/src/syncer.js @@ -37,8 +37,8 @@ function sync(dir, taskProcessor, concurrency, callback) { var curCacheIndex = 0, addQueue = [ ], delQueue = [ ]; - var cacheFile = path.join(paths.SNAPSHOT_DIR, path.basename(dir) + '.cache'), - newCacheFile = path.join(paths.SNAPSHOT_DIR, path.basename(dir) + '.cache.new'); + var cacheFile = path.join(paths.BACKUP_INFO_DIR, path.basename(dir) + '.sync.cache'), + newCacheFile = path.join(paths.BACKUP_INFO_DIR, path.basename(dir) + '.sync.cache.new'); var cache = readCache(cacheFile); @@ -104,4 +104,4 @@ function sync(dir, taskProcessor, concurrency, callback) { callback(); }); }); -} \ No newline at end of file +} diff --git a/src/test/setupTest b/src/test/setupTest index 1059e4b65..20366ac06 100755 --- a/src/test/setupTest +++ b/src/test/setupTest @@ -13,7 +13,7 @@ cd $HOME/.cloudron_test mkdir -p configs mkdir -p appsdata mkdir -p boxdata/appicons boxdata/mail boxdata/certs boxdata/mail/dkim/localhost boxdata/mail/dkim/foobar.com -mkdir -p platformdata/addons/mail platformdata/nginx/cert platformdata/nginx/applications platformdata/collectd/collectd.conf.d platformdata/addons platformdata/logrotate.d platformdata/snapshots +mkdir -p platformdata/addons/mail platformdata/nginx/cert platformdata/nginx/applications platformdata/collectd/collectd.conf.d platformdata/addons platformdata/logrotate.d platformdata/backup # put cert openssl req -x509 -newkey rsa:2048 -keyout platformdata/nginx/cert/host.key -out platformdata/nginx/cert/host.cert -days 3650 -subj '/CN=localhost' -nodes diff --git a/webadmin/src/views/settings.html b/webadmin/src/views/settings.html index 4608f8749..acc7ec719 100644 --- a/webadmin/src/views/settings.html +++ b/webadmin/src/views/settings.html @@ -379,7 +379,11 @@
-

{{ createBackup.message }}

+

{{ createBackup.message }} {{ createBackup.detail }}

+

+

{{ createBackup.result }}
+
Backup Successful
+

diff --git a/webadmin/src/views/settings.js b/webadmin/src/views/settings.js index f16cba991..25a8d72e7 100644 --- a/webadmin/src/views/settings.js +++ b/webadmin/src/views/settings.js @@ -117,14 +117,17 @@ angular.module('Application').controller('SettingsController', ['$scope', '$loca $scope.createBackup = { busy: false, - percent: 100, + percent: 0, message: '', errorMessage: '', + result: '', doCreateBackup: function () { $scope.createBackup.busy = true; $scope.createBackup.percent = 0; $scope.createBackup.message = ''; + $scope.createBackup.detail = ''; + $scope.createBackup.result = ''; $scope.createBackup.errorMessage = ''; Client.backup(function (error) { @@ -154,12 +157,16 @@ angular.module('Application').controller('SettingsController', ['$scope', '$loca $scope.createBackup.busy = false; $scope.createBackup.message = ''; + $scope.createBackup.detail = ''; + $scope.createBackup.percent = 100; // indicates that 'result' is valid + $scope.createBackup.result = data.backup ? data.backup.message : null; return fetchBackups(); } $scope.createBackup.percent = data.backup.percent; $scope.createBackup.message = data.backup.message; + $scope.createBackup.detail = data.backup.detail; window.setTimeout(checkIfDone, 500); }); }