jjkiers/cloudron-box
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Add tasks table and API

Browse Source 
progress will be tracked with this table instead of being in-process
like progress.js
This commit is contained in:
Girish Ramakrishnan
2018-11-16 11:13:03 -08:00
parent 390e69c01c
commit 218739a6b5
11 changed files with 259 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/backups.js
+21 -26
View File
@@ -56,7 +56,6 @@ var addons = require('./addons.js'),
once = require('once'),
path = require('path'),
paths = require('./paths.js'),
progress = require('./progress.js'),
progressStream = require('progress-stream'),
safe = require('safetydance'),
shell = require('./shell.js'),
@@ -64,6 +63,7 @@ var addons = require('./addons.js'),
superagent = require('superagent'),
syncer = require('./syncer.js'),
tar = require('tar-fs'),
tasks = require('./tasks.js'),
util = require('util'),
zlib = require('zlib');
@@ -186,11 +186,6 @@ function getBackupFilePath(backupConfig, backupId, format) {
}
}
function log(detail) {
debug(detail);
progress.setDetail(progress.BACKUP, detail);
}
function encryptFilePath(filePath, key) {
assert.strictEqual(typeof filePath, 'string');
assert.strictEqual(typeof key, 'string');
@@ -470,7 +465,7 @@ function restoreFsMetadata(appDataDir, callback) {
assert.strictEqual(typeof appDataDir, 'string');
assert.strictEqual(typeof callback, 'function');
log('Recreating empty directories');
tasks.setProgress(tasks.TASK_BACKUP, { detail: 'Recreating empty directories' }, NOOP_CALLBACK);
var metadataJson = safe.fs.readFileSync(path.join(appDataDir, 'fsmetadata.json'), 'utf8');
if (metadataJson === null) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, 'Error loading fsmetadata.txt:' + safe.error.message));
@@ -538,7 +533,7 @@ function download(backupConfig, backupId, format, dataDir, callback) {
assert.strictEqual(typeof dataDir, 'string');
assert.strictEqual(typeof callback, 'function');
log(`Downloading ${backupId} of format ${format} to ${dataDir}`);
tasks.setProgress(tasks.TASK_BACKUP, { detail: `Downloading ${backupId} of format ${format} to ${dataDir}` }, NOOP_CALLBACK);
if (format === 'tgz') {
api(backupConfig.provider).download(backupConfig, getBackupFilePath(backupConfig, backupId, format), function (error, sourceStream) {
@@ -642,7 +637,7 @@ function setSnapshotInfo(id, info, callback) {
function snapshotBox(callback) {
assert.strictEqual(typeof callback, 'function');
log('Snapshotting box');
tasks.setProgress(tasks.TASK_BACKUP, { detail: 'Snapshotting box' }, NOOP_CALLBACK);
database.exportToFile(`${paths.BOX_DATA_DIR}/box.mysqldump`, function (error) {
if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
@@ -711,13 +706,13 @@ function rotateBoxBackup(backupConfig, timestamp, appBackupIds, callback) {
var backupId = util.format('%s/box_%s_v%s', timestamp, snapshotTime, config.version());
const format = backupConfig.format;
log(`Rotating box backup to id ${backupId}`);
tasks.setProgress(tasks.TASK_BACKUP, { detail: `Rotating box backup to id ${backupId}` }, NOOP_CALLBACK);
backupdb.add({ id: backupId, version: config.version(), type: backupdb.BACKUP_TYPE_BOX, dependsOn: appBackupIds, manifest: null, format: format }, function (error) {
if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
var copy = api(backupConfig.provider).copy(backupConfig, getBackupFilePath(backupConfig, 'snapshot/box', format), getBackupFilePath(backupConfig, backupId, format));
copy.on('progress', log);
copy.on('progress', function (detail) { tasks.setProgress(tasks.TASK_BACKUP, { detail }, NOOP_CALLBACK); });
copy.on('done', function (copyBackupError) {
const state = copyBackupError ? backupdb.BACKUP_STATE_ERROR : backupdb.BACKUP_STATE_NORMAL;
@@ -725,7 +720,7 @@ function rotateBoxBackup(backupConfig, timestamp, appBackupIds, callback) {
if (copyBackupError) return callback(copyBackupError);
if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
log(`Rotated box backup successfully as id ${backupId}`);
tasks.setProgress(tasks.TASK_BACKUP, { detail: `Rotated box backup successfully as id ${backupId}` }, NOOP_CALLBACK);
backupDone(backupConfig, backupId, appBackupIds, function (error) {
if (error) return callback(error);
@@ -766,7 +761,7 @@ function snapshotApp(app, callback) {
assert.strictEqual(typeof app, 'object');
assert.strictEqual(typeof callback, 'function');
log(`Snapshotting app ${app.id}`);
tasks.setProgress(tasks.TASK_BACKUP, { detail: `Snapshotting app ${app.id}` }, NOOP_CALLBACK);
if (!safe.fs.writeFileSync(path.join(paths.APPS_DATA_DIR, app.id + '/config.json'), JSON.stringify(apps.getAppConfig(app)))) {
return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, 'Error creating config.json: ' + safe.error.message));
@@ -793,13 +788,13 @@ function rotateAppBackup(backupConfig, app, timestamp, callback) {
var backupId = util.format('%s/app_%s_%s_v%s', timestamp, app.id, snapshotTime, manifest.version);
const format = backupConfig.format;
log(`Rotating app backup of ${app.id} to id ${backupId}`);
tasks.setProgress(tasks.TASK_BACKUP, { detail: `Rotating app backup of ${app.id} to id ${backupId}` }, NOOP_CALLBACK);
backupdb.add({ id: backupId, version: manifest.version, type: backupdb.BACKUP_TYPE_APP, dependsOn: [ ], manifest: manifest, format: format }, function (error) {
if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
var copy = api(backupConfig.provider).copy(backupConfig, getBackupFilePath(backupConfig, `snapshot/app_${app.id}`, format), getBackupFilePath(backupConfig, backupId, format));
copy.on('progress', log);
copy.on('progress', function (detail) { tasks.setProgress(tasks.TASK_BACKUP, { detail }, NOOP_CALLBACK); });
copy.on('done', function (copyBackupError) {
const state = copyBackupError ? backupdb.BACKUP_STATE_ERROR : backupdb.BACKUP_STATE_NORMAL;
@@ -807,7 +802,7 @@ function rotateAppBackup(backupConfig, app, timestamp, callback) {
if (copyBackupError) return callback(copyBackupError);
if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
log(`Rotated app backup of ${app.id} successfully to id ${backupId}`);
tasks.setProgress(tasks.TASK_BACKUP, { detail: `Rotated app backup of ${app.id} successfully to id ${backupId}` }, NOOP_CALLBACK);
callback(null, backupId);
});
@@ -863,10 +858,10 @@ function backupApp(app, callback) {
const timestamp = (new Date()).toISOString().replace(/[T.]/g, '-').replace(/[:Z]/g,'');
progress.set(progress.BACKUP, 10, 'Backing up ' + app.fqdn);
tasks.setProgress(tasks.TASK_BACKUP, { precent: 10, mesage: `Backing up ${app.fqdn}` }, NOOP_CALLBACK);
backupAppWithTimestamp(app, timestamp, function (error) {
progress.set(progress.BACKUP, 100, error ? error.message : '');
tasks.setProgress(tasks.TASK_BACKUP, { percent: 100, result: error ? error.message : '' }, NOOP_CALLBACK);
callback(error);
});
@@ -889,12 +884,12 @@ function backupBoxAndApps(auditSource, callback) {
var step = 100/(allApps.length+2);
async.mapSeries(allApps, function iterator(app, iteratorCallback) {
progress.set(progress.BACKUP, step * processed, 'Backing up ' + app.fqdn);
tasks.setProgress(tasks.TASK_BACKUP, { percent: step * processed, message: `Backing up ${app.fqdn}` }, NOOP_CALLBACK);
++processed;
if (!app.enableBackup) {
progress.set(progress.BACKUP, step * processed, 'Skipped backup ' + app.fqdn);
tasks.setProgress(tasks.TASK_BACKUP, { percent: step * processed, message: `Skipped backup ${app.fqdn}` }, NOOP_CALLBACK);
return iteratorCallback(null, null); // nothing to backup
}
@@ -904,22 +899,22 @@ function backupBoxAndApps(auditSource, callback) {
return iteratorCallback(error);
}
progress.set(progress.BACKUP, step * processed, 'Backed up ' + app.fqdn);
tasks.setProgress(tasks.TASK_BACKUP, { percent: step * processed, message: `Backed up ${app.fqdn}` }, NOOP_CALLBACK);
iteratorCallback(null, backupId || null); // clear backupId if is in BAD_STATE and never backed up
});
}, function appsBackedUp(error, backupIds) {
if (error) {
progress.set(progress.BACKUP, 100, error.message);
tasks.setProgress(tasks.TASK_BACKUP, { percent: 100, result: error.message }, NOOP_CALLBACK);
return callback(error);
}
backupIds = backupIds.filter(function (id) { return id !== null; }); // remove apps in bad state that were never backed up
progress.set(progress.BACKUP, step * processed, 'Backing up system data');
tasks.setProgress(tasks.TASK_BACKUP, { percent: step * processed, message: 'Backing up system data' }, NOOP_CALLBACK);
backupBoxWithAppBackupIds(backupIds, timestamp, function (error, backupId) {
progress.set(progress.BACKUP, 100, error ? error.message : '');
tasks.setProgress(tasks.TASK_BACKUP, { percent: 100, result: error ? error.message : '' }, NOOP_CALLBACK);
eventlog.add(eventlog.ACTION_BACKUP_FINISH, auditSource, { errorMessage: error ? error.message : null, backupId: backupId, timestamp: timestamp });
@@ -937,7 +932,7 @@ function startBackupTask(auditSource, callback) {
if (error) return callback(new BackupsError(BackupsError.BAD_STATE, error.message));
let startTime = new Date();
progress.set(progress.BACKUP, 0, 'Starting'); // ensure tools can 'wait' on progress
tasks.setProgress(tasks.TASK_BACKUP, { percent: 0, message: 'Starting' }, NOOP_CALLBACK); // ensure tools can 'wait' on progress
let fd = safe.fs.openSync(paths.BACKUP_LOG_FILE, 'a'); // will autoclose
if (!fd) {
@@ -964,7 +959,7 @@ function startBackupTask(auditSource, callback) {
locker.unlock(locker.OP_FULL_BACKUP);
progress.set(progress.BACKUP, 100, error ? error.message : '');
tasks.setProgress(tasks.TASK_BACKUP, { percent: 100, result: error ? error.message : '' }, NOOP_CALLBACK);
if (error) mailer.backupFailed(error);
debug('startBackupTask: backup took %s seconds', (new Date() - startTime)/1000);
src/routes/backups.js
+1 -11
View File
@@ -2,8 +2,7 @@
exports = module.exports = {
list: list,
startBackup: startBackup,
stopBackup: stopBackup
startBackup: startBackup
};
var backupdb = require('../backupdb.js'),
@@ -42,12 +41,3 @@ function startBackup(req, res, next) {
next(new HttpSuccess(202, {}));
});
}
function stopBackup(req, res, next) {
backups.stopBackupTask(auditSource(req), function (error) {
if (error && error.reason === BackupsError.BAD_STATE) return next(new HttpError(409, error.message));
if (error) return next(new HttpError(500, error));
next(new HttpSuccess(202, {}));
});
}
src/routes/index.js
+1
View File
@@ -19,5 +19,6 @@ exports = module.exports = {
sysadmin: require('./sysadmin.js'),
settings: require('./settings.js'),
ssh: require('./ssh.js'),
tasks: require('./tasks.js'),
users: require('./users.js')
};
src/routes/tasks.js
+39
View File
@@ -0,0 +1,39 @@
'use strict';
exports = module.exports = {
getProgress: getProgress,
stopTask: stopTask
};
let assert = require('assert'),
HttpError = require('connect-lastmile').HttpError,
HttpSuccess = require('connect-lastmile').HttpSuccess,
TaskError = require('../tasks.js').TaskError,
tasks = require('../tasks.js');
function auditSource(req) {
var ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress || null;
return { ip: ip, username: req.user ? req.user.username : null, userId: req.user ? req.user.id : null };
}
function stopTask(req, res, next) {
assert.strictEqual(typeof req.params.taskId, 'string');
tasks.stopTask(req.params.taskId, auditSource(req), function (error) {
if (error && error.reason === TaskError.NOT_FOUND) return next(new HttpError(404, 'No such task'));
if (error) return next(new HttpError(500, error));
next(new HttpSuccess(204, {}));
});
}
function getProgress(req, res, next) {
assert.strictEqual(typeof req.params.taskId, 'string');
tasks.getProgress(req.params.taskId, function (error, progress) {
if (error && error.reason === TaskError.NOT_FOUND) return next(new HttpError(404, 'No such task'));
if (error) return next(new HttpError(500, error));
next(new HttpSuccess(200, progress));
});
}
src/routes/test/backups-test.js
+3 -3
View File
@@ -84,7 +84,7 @@ describe('Backups API', function () {
describe('create', function () {
it('fails due to mising token', function (done) {
superagent.post(SERVER_URL + '/api/v1/cloudron/start_backup')
superagent.post(SERVER_URL + '/api/v1/backups')
.end(function (error, result) {
expect(result.statusCode).to.equal(401);
done();
@@ -92,7 +92,7 @@ describe('Backups API', function () {
});
it('fails due to wrong token', function (done) {
superagent.post(SERVER_URL + '/api/v1/cloudron/start_backup')
superagent.post(SERVER_URL + '/api/v1/backups')
.query({ access_token: token.toUpperCase() })
.end(function (error, result) {
expect(result.statusCode).to.equal(401);
@@ -101,7 +101,7 @@ describe('Backups API', function () {
});
it('succeeds', function (done) {
superagent.post(SERVER_URL + '/api/v1/cloudron/start_backup')
superagent.post(SERVER_URL + '/api/v1/backups')
.query({ access_token: token })
.end(function (error, result) {
expect(result.statusCode).to.equal(202);
src/server.js
+4 -2
View File
@@ -134,11 +134,13 @@ function initializeExpressSync() {
router.del ('/api/v1/cloudron/ssh/authorized_keys/:identifier', cloudronScope, isUnmanaged, routes.ssh.delAuthorizedKey);
router.get ('/api/v1/cloudron/eventlog', cloudronScope, routes.eventlog.get);
router.post('/api/v1/cloudron/start_backup', settingsScope, routes.backups.startBackup);
router.post('/api/v1/cloudron/stop_backup', settingsScope, routes.backups.stopBackup);
// tasks
router.get ('/api/v1/tasks/:taskId', settingsScope, routes.tasks.getProgress);
router.post('/api/v1/tasks/:taskId/stop', settingsScope, routes.tasks.stopTask);
// backups
router.get ('/api/v1/backups', settingsScope, routes.backups.list);
router.post('/api/v1/backups', settingsScope, routes.backups.startBackup);
// config route (for dashboard)
router.get ('/api/v1/config', profileScope, routes.cloudron.getConfig);
src/taskdb.js
+51
View File
@@ -0,0 +1,51 @@
'use strict';
exports = module.exports = {
setProgress: setProgress,
getProgress: getProgress
};
let assert = require('assert'),
database = require('./database.js'),
DatabaseError = require('./databaseerror'),
_ = require('underscore');
const TASKS_FIELDS = [ 'id', 'percent', 'message', 'detail', 'creationTime', 'result', 'ts' ];
function setProgress(id, progress, callback) {
assert.strictEqual(typeof id, 'string');
assert.strictEqual(typeof progress, 'object');
assert.strictEqual(typeof callback, 'function');
let data = _.extend({ id: id }, progress);
let keys = [ ],
questionMarks = Array(Object.keys(data).length).fill('?').join(','),
fields = [ ], values = [ ];
for (var f in data) {
keys.push(f);
fields.push(`${f} = ?`);
values.push(data[f]); // for the INSERT fields
}
values = values.concat(values); // for the UPDATE fields
database.query(`INSERT INTO tasks (${keys.join(', ')}) VALUES (${questionMarks}) ON DUPLICATE KEY UPDATE ${fields}`, values, function (error) {
if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
callback(null);
});
}
function getProgress(id, callback) {
assert.strictEqual(typeof id, 'string');
assert.strictEqual(typeof callback, 'function');
database.query('SELECT ' + TASKS_FIELDS + ' FROM tasks WHERE id = ?', [ id ], function (error, result) {
if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
callback(null, result[0]);
});
}
src/tasks.js
+89
View File
@@ -0,0 +1,89 @@
'use strict';
exports = module.exports = {
setProgress: setProgress,
getProgress: getProgress,
stopTask: stopTask,
TaskError: TaskError,
TASK_BACKUP: 'backup'
};
let assert = require('assert'),
BackupsError = require('./backups.js').BackupsError,
backups = require('./backups.js'),
DatabaseError = require('./databaseerror.js'),
debug = require('debug')('box:tasks'),
taskdb = require('./taskdb.js'),
util = require('util');
function TaskError(reason, errorOrMessage) {
assert.strictEqual(typeof reason, 'string');
assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined');
Error.call(this);
Error.captureStackTrace(this, this.constructor);
this.name = this.constructor.name;
this.reason = reason;
if (typeof errorOrMessage === 'undefined') {
this.message = reason;
} else if (typeof errorOrMessage === 'string') {
this.message = errorOrMessage;
} else {
this.message = 'Internal error';
this.nestedError = errorOrMessage;
}
}
util.inherits(TaskError, Error);
TaskError.INTERNAL_ERROR = 'Internal Error';
TaskError.BAD_STATE = 'Bad State';
TaskError.NOT_FOUND = 'Not Found';
function setProgress(id, progress, callback) {
assert.strictEqual(typeof id, 'string');
assert.strictEqual(typeof progress, 'object');
assert.strictEqual(typeof callback, 'function');
debug(`${id}: ${JSON.stringify(progress)}`);
taskdb.setProgress(id, progress, function (error) {
if (error) return callback(new TaskError(TaskError.INTERNAL_ERROR, error));
callback();
});
}
function getProgress(id, callback) {
assert.strictEqual(typeof id, 'string');
assert.strictEqual(typeof callback, 'function');
taskdb.getProgress(id, function (error, progress) {
if (error && error.reason == DatabaseError.NOT_FOUND) return callback(new TaskError(TaskError.NOT_FOUND));
if (error) return callback(new TaskError(TaskError.INTERNAL_ERROR, error));
callback(null, progress);
});
}
function stopTask(id, auditSource, callback) {
assert.strictEqual(typeof id, 'string');
assert.strictEqual(typeof auditSource, 'object');
assert.strictEqual(typeof callback, 'function');
switch (id) {
case exports.TASK_BACKUP:
backups.stopBackupTask(auditSource, function (error) {
if (error && error.reason === BackupsError.BAD_STATE) return callback(new TaskError(TaskError.NOT_FOUND));
if (error) return callback(new TaskError(TaskError.INTERNAL_ERROR, error));
callback(null);
});
break;
default:
return callback(new TaskError(TaskError.NOT_FOUND));
}
}
src/test/backups-test.js
+13 -9
View File
@@ -21,7 +21,8 @@ var async = require('async'),
progress = require('../progress.js'),
rimraf = require('rimraf'),
settings = require('../settings.js'),
SettingsError = require('../settings.js').SettingsError;
SettingsError = require('../settings.js').SettingsError,
tasks = require('../tasks.js');
function compareDirectories(one, two, callback) {
readdirp({ root: one }, function (error, treeOne) {
@@ -73,16 +74,19 @@ function createBackup(callback) {
if (error) return callback(error);
function waitForBackup() {
var p = progress.getAll();
if (p.backup.percent !== 100) return setTimeout(waitForBackup, 1000);
if (p.backup.message) return callback(new Error('backup failed:' + p.backup.message));
backups.getByStatePaged(backupdb.BACKUP_STATE_NORMAL, 1, 1, function (error, result) {
tasks.getProgress(tasks.TASK_BACKUP, function (error, p) {
if (error) return callback(error);
if (result.length !== 1) return callback(new Error('result is not of length 1'));
callback(null, result[0]);
if (p.percent !== 100) return setTimeout(waitForBackup, 1000);
if (p.result) return callback(new Error('backup failed:' + p.result));
backups.getByStatePaged(backupdb.BACKUP_STATE_NORMAL, 1, 1, function (error, result) {
if (error) return callback(error);
if (result.length !== 1) return callback(new Error('result is not of length 1'));
callback(null, result[0]);
});
});
}