jjkiers/cloudron-box
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

apptask: asyncify

Browse Source
This commit is contained in:
Girish Ramakrishnan
2021-09-16 13:59:03 -07:00
parent b5539120f1
commit 9a2ed4f2c8
9 changed files with 505 additions and 744 deletions
Download Patch File Download Diff File Expand all files Collapse all files

242
src/backuptask.js
View File

@@ -645,54 +645,41 @@ function download(backupConfig, backupId, format, dataLayout, progressCallback,
}
}
function restore(backupConfig, backupId, progressCallback, callback) {
async function restore(backupConfig, backupId, progressCallback) {
assert.strictEqual(typeof backupConfig, 'object');
assert.strictEqual(typeof backupId, 'string');
assert.strictEqual(typeof progressCallback, 'function');
assert.strictEqual(typeof callback, 'function');
const boxDataDir = safe.fs.realpathSync(paths.BOX_DATA_DIR);
if (!boxDataDir) return callback(new BoxError(BoxError.FS_ERROR, `Error resolving boxdata: ${safe.error.message}`));
if (!boxDataDir) throw new BoxError(BoxError.FS_ERROR, `Error resolving boxdata: ${safe.error.message}`);
const dataLayout = new DataLayout(boxDataDir, []);
download(backupConfig, backupId, backupConfig.format, dataLayout, progressCallback, function (error) {
if (error) return callback(error);
await util.promisify(download)(backupConfig, backupId, backupConfig.format, dataLayout, progressCallback);
debug('restore: download completed, importing database');
debug('restore: download completed, importing database');
database.importFromFile(`${dataLayout.localRoot()}/box.mysqldump`, async function (error) {
if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
await database.importFromFile(`${dataLayout.localRoot()}/box.mysqldump`);
debug('restore: database imported');
debug('restore: database imported');
[error] = await safe(settings.initCache());
callback(error);
});
});
await settings.initCache();
}
function downloadApp(app, restoreConfig, progressCallback, callback) {
async function downloadApp(app, restoreConfig, progressCallback) {
assert.strictEqual(typeof app, 'object');
assert.strictEqual(typeof restoreConfig, 'object');
assert.strictEqual(typeof progressCallback, 'function');
assert.strictEqual(typeof callback, 'function');
const appDataDir = safe.fs.realpathSync(path.join(paths.APPS_DATA_DIR, app.id));
if (!appDataDir) return callback(new BoxError(BoxError.FS_ERROR, safe.error.message));
if (!appDataDir) throw new BoxError(BoxError.FS_ERROR, safe.error.message);
const dataLayout = new DataLayout(appDataDir, app.dataDir ? [{ localDir: app.dataDir, remoteDir: 'data' }] : []);
const startTime = new Date();
const getBackupConfigFunc = restoreConfig.backupConfig ? (next) => next(null, restoreConfig.backupConfig) : getBackupConfig;
const backupConfig = restoreConfig.backupConfig || await settings.getBackupConfig();
getBackupConfigFunc(function (error, backupConfig) {
if (error) return callback(error);
download(backupConfig, restoreConfig.backupId, restoreConfig.backupFormat, dataLayout, progressCallback, function (error) {
debug('downloadApp: time: %s', (new Date() - startTime)/1000);
callback(error);
});
});
const downloadAsync = util.promisify(download);
await downloadAsync(backupConfig, restoreConfig.backupId, restoreConfig.backupFormat, dataLayout, progressCallback);
debug('downloadApp: time: %s', (new Date() - startTime)/1000);
}
function runBackupUpload(uploadConfig, progressCallback, callback) {
@@ -731,53 +718,42 @@ function runBackupUpload(uploadConfig, progressCallback, callback) {
});
}
function snapshotBox(progressCallback, callback) {
async function snapshotBox(progressCallback) {
assert.strictEqual(typeof progressCallback, 'function');
assert.strictEqual(typeof callback, 'function');
progressCallback({ message: 'Snapshotting box' });
const startTime = new Date();
database.exportToFile(`${paths.BOX_DATA_DIR}/box.mysqldump`, function (error) {
if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
debug(`snapshotBox: took ${(new Date() - startTime)/1000} seconds`);
return callback();
});
await database.exportToFile(`${paths.BOX_DATA_DIR}/box.mysqldump`);
debug(`snapshotBox: took ${(new Date() - startTime)/1000} seconds`);
}
function uploadBoxSnapshot(backupConfig, progressCallback, callback) {
async function uploadBoxSnapshot(backupConfig, progressCallback) {
assert.strictEqual(typeof backupConfig, 'object');
assert.strictEqual(typeof progressCallback, 'function');
assert.strictEqual(typeof callback, 'function');
snapshotBox(progressCallback, function (error) {
if (error) return callback(error);
await snapshotBox(progressCallback);
const boxDataDir = safe.fs.realpathSync(paths.BOX_DATA_DIR);
if (!boxDataDir) return callback(new BoxError(BoxError.FS_ERROR, `Error resolving boxdata: ${safe.error.message}`));
const boxDataDir = safe.fs.realpathSync(paths.BOX_DATA_DIR);
if (!boxDataDir) throw new BoxError(BoxError.FS_ERROR, `Error resolving boxdata: ${safe.error.message}`);
const uploadConfig = {
backupId: 'snapshot/box',
backupConfig,
dataLayout: new DataLayout(boxDataDir, []),
progressTag: 'box'
};
const uploadConfig = {
backupId: 'snapshot/box',
backupConfig,
dataLayout: new DataLayout(boxDataDir, []),
progressTag: 'box'
};
progressCallback({ message: 'Uploading box snapshot' });
progressCallback({ message: 'Uploading box snapshot' });
const startTime = new Date();
const startTime = new Date();
runBackupUpload(uploadConfig, progressCallback, function (error) {
if (error) return callback(error);
await util.promisify(runBackupUpload)(uploadConfig, progressCallback);
debug(`uploadBoxSnapshot: took ${(new Date() - startTime)/1000} seconds`);
debug(`uploadBoxSnapshot: took ${(new Date() - startTime)/1000} seconds`);
backups.setSnapshotInfo('box', { timestamp: new Date().toISOString(), format: backupConfig.format }, callback);
});
});
await backups.setSnapshotInfo('box', { timestamp: new Date().toISOString(), format: backupConfig.format });
}
async function rotateBoxBackup(backupConfig, tag, options, appBackupIds, progressCallback) {
@@ -822,23 +798,18 @@ async function rotateBoxBackup(backupConfig, tag, options, appBackupIds, progres
});
}
function backupBoxWithAppBackupIds(appBackupIds, tag, options, progressCallback, callback) {
async function backupBoxWithAppBackupIds(appBackupIds, tag, options, progressCallback) {
assert(Array.isArray(appBackupIds));
assert.strictEqual(typeof tag, 'string');
assert.strictEqual(typeof options, 'object');
assert.strictEqual(typeof progressCallback, 'function');
assert.strictEqual(typeof callback, 'function');
getBackupConfig(function (error, backupConfig) {
if (error) return callback(error);
const backupConfig = await settings.getBackupConfig();
uploadBoxSnapshot(backupConfig, progressCallback, async function (error) {
if (error) return callback(error);
await uploadBoxSnapshot(backupConfig, progressCallback);
const [rotateError, backupId] = await safe(rotateBoxBackup(backupConfig, tag, options, appBackupIds, progressCallback));
callback(rotateError, backupId);
});
});
const backupId = await rotateBoxBackup(backupConfig, tag, options, appBackupIds, progressCallback);
return backupId;
}
async function rotateAppBackup(backupConfig, app, tag, options, progressCallback) {
@@ -888,150 +859,115 @@ async function rotateAppBackup(backupConfig, app, tag, options, progressCallback
});
}
function backupApp(app, options, progressCallback, callback) {
async function backupApp(app, options, progressCallback) {
assert.strictEqual(typeof app, 'object');
assert.strictEqual(typeof options, 'object');
assert.strictEqual(typeof progressCallback, 'function');
assert.strictEqual(typeof callback, 'function');
if (options.snapshotOnly) return snapshotApp(app, progressCallback, callback);
if (options.snapshotOnly) return await snapshotApp(app, progressCallback);
const tag = (new Date()).toISOString().replace(/[T.]/g, '-').replace(/[:Z]/g,'');
debug(`backupApp - Backing up ${app.fqdn} with tag ${tag}`);
backupAppWithTag(app, tag, options, progressCallback, callback);
await backupAppWithTag(app, tag, options, progressCallback);
}
function snapshotApp(app, progressCallback, callback) {
async function snapshotApp(app, progressCallback) {
assert.strictEqual(typeof app, 'object');
assert.strictEqual(typeof progressCallback, 'function');
assert.strictEqual(typeof callback, 'function');
const startTime = new Date();
progressCallback({ message: `Snapshotting app ${app.fqdn}` });
const appsBackupConfig = util.callbackify(apps.backupConfig);
await apps.backupConfig(app);
await services.backupAddons(app, app.manifest.addons);
appsBackupConfig(app, async function (error) {
if (error) return callback(error);
[error] = await safe(services.backupAddons(app, app.manifest.addons));
if (error) return callback(error);
debugApp(app, `snapshotApp: took ${(new Date() - startTime)/1000} seconds`);
return callback(null);
});
debugApp(app, `snapshotApp: took ${(new Date() - startTime)/1000} seconds`);
}
function uploadAppSnapshot(backupConfig, app, progressCallback, callback) {
async function uploadAppSnapshot(backupConfig, app, progressCallback) {
assert.strictEqual(typeof backupConfig, 'object');
assert.strictEqual(typeof app, 'object');
assert.strictEqual(typeof progressCallback, 'function');
assert.strictEqual(typeof callback, 'function');
snapshotApp(app, progressCallback, function (error) {
if (error) return callback(error);
await snapshotApp(app, progressCallback);
const backupId = util.format('snapshot/app_%s', app.id);
const appDataDir = safe.fs.realpathSync(path.join(paths.APPS_DATA_DIR, app.id));
if (!appDataDir) return callback(new BoxError(BoxError.FS_ERROR, `Error resolving appsdata: ${safe.error.message}`));
const backupId = util.format('snapshot/app_%s', app.id);
const appDataDir = safe.fs.realpathSync(path.join(paths.APPS_DATA_DIR, app.id));
if (!appDataDir) throw new BoxError(BoxError.FS_ERROR, `Error resolving appsdata: ${safe.error.message}`);
const dataLayout = new DataLayout(appDataDir, app.dataDir ? [{ localDir: app.dataDir, remoteDir: 'data' }] : []);
const dataLayout = new DataLayout(appDataDir, app.dataDir ? [{ localDir: app.dataDir, remoteDir: 'data' }] : []);
progressCallback({ message: `Uploading app snapshot ${app.fqdn}`});
progressCallback({ message: `Uploading app snapshot ${app.fqdn}`});
const uploadConfig = {
backupId,
backupConfig,
dataLayout,
progressTag: app.fqdn
};
const uploadConfig = {
backupId,
backupConfig,
dataLayout,
progressTag: app.fqdn
};
const startTime = new Date();
const startTime = new Date();
runBackupUpload(uploadConfig, progressCallback, function (error) {
if (error) return callback(error);
await util.promisify(runBackupUpload)(uploadConfig, progressCallback);
debugApp(app, `uploadAppSnapshot: ${backupId} done. ${(new Date() - startTime)/1000} seconds`);
debugApp(app, `uploadAppSnapshot: ${backupId} done. ${(new Date() - startTime)/1000} seconds`);
backups.setSnapshotInfo(app.id, { timestamp: new Date().toISOString(), manifest: app.manifest, format: backupConfig.format }, callback);
});
});
await backups.setSnapshotInfo(app.id, { timestamp: new Date().toISOString(), manifest: app.manifest, format: backupConfig.format });
}
async function backupAppWithTag(app, tag, options, progressCallback, callback) {
async function backupAppWithTag(app, tag, options, progressCallback) {
assert.strictEqual(typeof app, 'object');
assert.strictEqual(typeof tag, 'string');
assert.strictEqual(typeof options, 'object');
assert.strictEqual(typeof progressCallback, 'function');
assert.strictEqual(typeof callback, 'function');
if (!canBackupApp(app)) { // if we cannot backup, reuse it's most recent backup
const [error, results] = await safe(backups.getByIdentifierAndStatePaged(app.id, backups.BACKUP_STATE_NORMAL, 1, 1));
if (error) return callback(error);
if (results.length === 0) return callback(null, null); // no backup to re-use
const results = await backups.getByIdentifierAndStatePaged(app.id, backups.BACKUP_STATE_NORMAL, 1, 1);
if (results.length === 0) return null; // no backup to re-use
return callback(null, results[0].id);
return results[0].id;
}
getBackupConfig(function (error, backupConfig) {
if (error) return callback(error);
const backupConfig = await settings.getBackupConfig();
uploadAppSnapshot(backupConfig, app, progressCallback, async function (error) {
if (error) return callback(error);
const [rotateError, backupId] = await safe(rotateAppBackup(backupConfig, app, tag, options, progressCallback));
callback(rotateError, backupId);
});
});
await uploadAppSnapshot(backupConfig, app, progressCallback);
const backupId = await rotateAppBackup(backupConfig, app, tag, options, progressCallback);
return backupId;
}
// this function expects you to have a lock. Unlike other progressCallback this also has a progress field
function backupBoxAndApps(options, progressCallback, callback) {
async function backupBoxAndApps(options, progressCallback) {
assert.strictEqual(typeof options, 'object');
assert.strictEqual(typeof progressCallback, 'function');
assert.strictEqual(typeof callback, 'function');
const tag = (new Date()).toISOString().replace(/[T.]/g, '-').replace(/[:Z]/g,'');
util.callbackify(apps.list)(function (error, allApps) {
if (error) return callback(error);
const allApps = await apps.list();
let percent = 1;
let step = 100/(allApps.length+2);
let percent = 1;
let step = 100/(allApps.length+2);
async.mapSeries(allApps, function iterator(app, iteratorCallback) {
progressCallback({ percent: percent, message: `Backing up ${app.fqdn}` });
percent += step;
const appBackupIds = [];
for (const app of allApps) {
progressCallback({ percent: percent, message: `Backing up ${app.fqdn}` });
percent += step;
if (!app.enableBackup) {
debug(`Skipped backup ${app.fqdn}`);
return iteratorCallback(null, null); // nothing to backup
}
if (!app.enableBackup) {
debug(`Skipped backup ${app.fqdn}`);
return; // nothing to backup
}
const startTime = new Date();
backupAppWithTag(app, tag, options, (progress) => progressCallback({ percent: percent, message: progress.message }), function (error, backupId) {
if (error) {
debugApp(app, 'Unable to backup', error);
return iteratorCallback(error);
}
const startTime = new Date();
const backupId = await backupAppWithTag(app, tag, options, (progress) => progressCallback({ percent: percent, message: progress.message }));
debugApp(app, `Backed up. Took ${(new Date() - startTime)/1000} seconds`);
if (backupId) appBackupIds.push(backupId); // backupId can be null if in BAD_STATE and never backed up
}
debugApp(app, `Backed up. Took ${(new Date() - startTime)/1000} seconds`);
progressCallback({ percent: percent, message: 'Backing up system data' });
percent += step;
iteratorCallback(null, backupId || null); // clear backupId if is in BAD_STATE and never backed up
});
}, function appsBackedUp(error, backupIds) {
if (error) return callback(error);
backupIds = backupIds.filter(function (id) { return id !== null; }); // remove apps in bad state that were never backed up
progressCallback({ percent: percent, message: 'Backing up system data' });
percent += step;
backupBoxWithAppBackupIds(backupIds, tag, options, (progress) => progressCallback({ percent: percent, message: progress.message }), callback);
});
});
const backupId = await backupBoxWithAppBackupIds(appBackupIds, tag, options, (progress) => progressCallback({ percent: percent, message: progress.message }));
return backupId;
}