jjkiers/cloudron-box
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Bring the caas storage backend up to speed

Browse Source 
This is mostly the same as the s3 backend at the moment
This commit is contained in:
Johannes Zellner
2017-04-19 13:30:02 +02:00
parent 65af062c33
commit 40f5d0412b
1 changed files with 217 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files

308
src/storage/caas.js
View File

@@ -1,29 +1,37 @@
'use strict';
exports = module.exports = {
getBoxBackupDetails: getBoxBackupDetails,
getAppBackupDetails: getAppBackupDetails,
getRestoreUrl: getRestoreUrl,
getAppRestoreConfig: getAppRestoreConfig,
getLocalFilePath: getLocalFilePath,
copyObject: copyObject,
backup: backup,
restore: restore,
copyBackup: copyBackup,
removeBackup: removeBackup,
getDownloadStream: getDownloadStream,
backupDone: backupDone,
testConfig: testConfig
testConfig: testConfig,
};
var assert = require('assert'),
var archiver = require('archiver'),
assert = require('assert'),
async = require('async'),
AWS = require('aws-sdk'),
BackupsError = require('../backups.js').BackupsError,
config = require('../config.js'),
crypto = require('crypto'),
debug = require('debug')('box:storage/caas'),
safe = require('safetydance'),
mkdirp = require('mkdirp'),
once = require('once'),
path = require('path'),
SettingsError = require('../settings.js').SettingsError,
superagent = require('superagent');
superagent = require('superagent'),
tar = require('tar-fs'),
zlib = require('zlib');
var FILE_TYPE = '.tar.gz';
// internal only
function getBackupCredentials(apiConfig, callback) {
assert.strictEqual(typeof apiConfig, 'object');
assert.strictEqual(typeof callback, 'function');
@@ -49,121 +57,180 @@ function getBackupCredentials(apiConfig, callback) {
});
}
function getBoxBackupDetails(apiConfig, id, callback) {
function getBackupFilePath(apiConfig, backupId) {
assert.strictEqual(typeof apiConfig, 'object');
assert.strictEqual(typeof id, 'string');
assert.strictEqual(typeof callback, 'function');
assert.strictEqual(typeof backupId, 'string');
getBackupCredentials(apiConfig, function (error, result) {
if (error) return callback(error);
var s3Url = 's3://' + apiConfig.bucket + '/' + apiConfig.prefix + '/' + id;
var region = apiConfig.region || 'us-east-1';
var details = {
backupScriptArguments: [ 's3', s3Url, result.accessKeyId, result.secretAccessKey, region, '', apiConfig.key, result.sessionToken ]
};
callback(null, details);
});
return path.join(apiConfig.prefix, backupId.endsWith(FILE_TYPE) ? backupId : backupId+FILE_TYPE);
}
function getAppBackupDetails(apiConfig, appId, dataId, configId, callback) {
// storage api
function backup(apiConfig, backupId, sourceDirectories, callback) {
assert.strictEqual(typeof apiConfig, 'object');
assert.strictEqual(typeof appId, 'string');
assert.strictEqual(typeof dataId, 'string');
assert.strictEqual(typeof configId, 'string');
assert.strictEqual(typeof backupId, 'string');
assert(Array.isArray(sourceDirectories));
assert.strictEqual(typeof callback, 'function');
getBackupCredentials(apiConfig, function (error, result) {
if (error) return callback(error);
callback = once(callback);
var s3DataUrl = 's3://' + apiConfig.bucket + '/' + apiConfig.prefix + '/' + dataId;
var s3ConfigUrl = 's3://' + apiConfig.bucket + '/' + apiConfig.prefix + '/' + configId;
var region = apiConfig.region || 'us-east-1';
var backupFilePath = getBackupFilePath(apiConfig, backupId);
var details = {
backupScriptArguments: [ 's3', appId, s3ConfigUrl, s3DataUrl, result.accessKeyId, result.secretAccessKey, region, '', apiConfig.key, result.sessionToken ]
};
callback(null, details);
});
}
function getRestoreUrl(apiConfig, filename, callback) {
assert.strictEqual(typeof apiConfig, 'object');
assert.strictEqual(typeof filename, 'string');
assert.strictEqual(typeof callback, 'function');
if (!apiConfig.bucket || !apiConfig.prefix) return new Error('Invalid configuration'); // prevent error in s3
debug('[%s] backup: %j -> %s', backupId, sourceDirectories, backupFilePath);
getBackupCredentials(apiConfig, function (error, credentials) {
if (error) return callback(error);
credentials.region = apiConfig.region; // use same region as where we uploaded
var s3 = new AWS.S3(credentials);
var archive = archiver('tar', { gzip: true });
var encrypt = crypto.createCipher('aes-256-cbc', apiConfig.key || '');
encrypt.on('error', function (error) {
console.error('[%s] backup: cipher stream error.', backupId, error);
});
archive.on('error', function (error) {
console.error('[%s] backup: archive stream error.', backupId, error);
});
archive.pipe(encrypt);
sourceDirectories.forEach(function (directory) {
// archive does not like destination beginning with a slash
directory.destination = path.normalize(directory.destination).replace(/^\//, '');
archive.directory(directory.source, directory.destination);
});
archive.finalize();
var params = {
Bucket: apiConfig.bucket,
Key: apiConfig.prefix + '/' + filename,
Expires: 60 * 60 * 24 /* 1 day */
Key: backupFilePath,
Body: encrypt
};
var url = s3.getSignedUrl('getObject', params);
var s3 = new AWS.S3(credentials);
s3.upload(params, function (error) {
if (error) {
console.error('[%s] backup: s3 upload error.', backupId, error);
return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error));
}
callback(null, { url: url });
});
}
function getAppRestoreConfig(apiConfig, backupId, callback) {
assert.strictEqual(typeof apiConfig, 'object');
assert.strictEqual(typeof backupId, 'string');
assert.strictEqual(typeof callback, 'function');
var configFilename = backupId.replace(/\.tar\.gz$/, '.json');
getRestoreUrl(apiConfig, configFilename, function (error, result) {
if (error) return callback(error);
superagent.get(result.url).buffer(true).timeout(30 * 1000).end(function (error, response) {
if (error && !error.response) return callback(new Error(error.message));
if (response.statusCode !== 200) return callback(new Error('Invalid response code when getting config.json : ' + response.statusCode));
var config = safe.JSON.parse(response.text);
if (!config) return callback(new Error('Error in config:' + safe.error.message));
return callback(null, config);
callback(null);
});
});
}
function getLocalFilePath(apiConfig, filename, callback) {
function restore(apiConfig, backupId, destinationDirectories, callback) {
assert.strictEqual(typeof apiConfig, 'object');
assert.strictEqual(typeof filename, 'string');
assert.strictEqual(typeof backupId, 'string');
assert(Array.isArray(destinationDirectories));
assert.strictEqual(typeof callback, 'function');
callback(new Error('not supported'));
var backupFilePath = getBackupFilePath(apiConfig, backupId);
debug('[%s] restore: %s -> %j', backupId, backupFilePath, destinationDirectories);
getBackupCredentials(apiConfig, function (error, credentials) {
if (error) return callback(error);
async.eachSeries(destinationDirectories, function (directory, callback) {
debug('[%s] restore: directory %s -> %s', backupId, directory.source, directory.destination);
// tar-fs reports without slash at the beginning
directory.source = path.normalize(directory.source).replace(/^\//, '');
mkdirp(directory.destination, function (error) {
if (error) return callback(error);
var params = {
Bucket: apiConfig.bucket,
Key: backupFilePath
};
var s3 = new AWS.S3(credentials);
var s3get = s3.getObject(params).createReadStream();
var decrypt = crypto.createDecipher('aes-256-cbc', apiConfig.key || '');
var gunzip = zlib.createGunzip({});
var IGNORE_PREFIX = '__ignore__';
var extract = tar.extract(directory.destination, {
ignore: function (name, header) { return header.name.startsWith(IGNORE_PREFIX); },
map: function (header) {
// ignore is called after map, we mark everything we dont want!
// else slice off the mapping prefix
if (!header.name.startsWith(directory.source)) header.name = IGNORE_PREFIX + header.name;
else header.name = header.name.slice(directory.source.length);
return header;
}
});
s3get.on('error', function (error) {
// TODO ENOENT for the mock, fix upstream!
if (error.code === 'NoSuchKey' || error.code === 'ENOENT') return callback(new BackupsError(BackupsError.NOT_FOUND));
console.error('[%s] restore: s3 stream error.', backupId, error);
callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error));
});
decrypt.on('error', function (error) {
console.error('[%s] restore: decipher stream error.', error);
callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
});
gunzip.on('error', function (error) {
console.error('[%s] restore: gunzip stream error.', error);
callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
});
extract.on('error', function (error) {
console.error('[%s] restore: extract stream error.', error);
callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
});
extract.on('finish', function () {
debug('[%s] restore: directory %s done.', backupId, directory.source);
callback();
});
s3get.pipe(decrypt).pipe(gunzip).pipe(extract);
});
}, function (error) {
if (error) return callback(error);
debug('[%s] restore: done', backupId);
callback(null);
});
});
}
function copyObject(apiConfig, from, to, callback) {
function copyBackup(apiConfig, oldBackupId, newBackupId, callback) {
assert.strictEqual(typeof apiConfig, 'object');
assert.strictEqual(typeof from, 'string');
assert.strictEqual(typeof to, 'string');
assert.strictEqual(typeof oldBackupId, 'string');
assert.strictEqual(typeof newBackupId, 'string');
assert.strictEqual(typeof callback, 'function');
if (!apiConfig.bucket || !apiConfig.prefix) return new Error('Invalid configuration'); // prevent error in s3
getBackupCredentials(apiConfig, function (error, credentials) {
if (error) return callback(error);
var params = {
Bucket: apiConfig.bucket, // target bucket
Key: apiConfig.prefix + '/' + to, // target file
CopySource: apiConfig.bucket + '/' + apiConfig.prefix + '/' + from, // source
Bucket: apiConfig.bucket,
Key: getBackupFilePath(apiConfig, newBackupId),
CopySource: path.join(apiConfig.bucket, getBackupFilePath(apiConfig, oldBackupId))
};
var s3 = new AWS.S3(credentials);
s3.copyObject(params, callback);
s3.copyObject(params, function (error) {
if (error && error.code === 'NoSuchKey') return callback(new BackupsError(BackupsError.NOT_FOUND));
if (error) {
console.error('copyBackup: s3 copy error.', error);
return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error));
}
callback(null);
});
});
}
@@ -173,9 +240,68 @@ function removeBackup(apiConfig, backupId, appBackupIds, callback) {
assert(Array.isArray(appBackupIds));
assert.strictEqual(typeof callback, 'function');
// Result: none
getBackupCredentials(apiConfig, function (error, credentials) {
if (error) return callback(error);
callback(new Error('not implemented'));
var params = {
Bucket: apiConfig.bucket,
Key: getBackupFilePath(apiConfig, backupId)
};
var s3 = new AWS.S3(credentials);
s3.deleteObject(params, function (error) {
if (error) console.error('Unable to remove %s. Not fatal.', params.Key, error);
callback(null);
});
});
}
function getDownloadStream(apiConfig, backupId, callback) {
assert.strictEqual(typeof apiConfig, 'object');
assert.strictEqual(typeof backupId, 'string');
assert.strictEqual(typeof callback, 'function');
callback = once(callback);
var backupFilePath = getBackupFilePath(apiConfig, backupId);
debug('[%s] getDownloadStream: %s %s', backupId, backupId, backupFilePath);
getBackupCredentials(apiConfig, function (error, credentials) {
if (error) return callback(error);
var params = {
Bucket: apiConfig.bucket,
Key: backupFilePath
};
var s3 = new AWS.S3(credentials);
s3.headObject(params, function (error) {
// TODO ENOENT for the mock, fix upstream!
if (error && (error.code === 'NotFound' || error.code === 'ENOENT')) return callback(new BackupsError(BackupsError.NOT_FOUND));
if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error));
var s3get = s3.getObject(params).createReadStream();
var decrypt = crypto.createDecipher('aes-256-cbc', apiConfig.key || '');
s3get.on('error', function (error) {
if (error.code === 'NoSuchKey') return callback(new BackupsError(BackupsError.NOT_FOUND));
console.error('[%s] getDownloadStream: s3 stream error.', backupId, error);
callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error));
});
decrypt.on('error', function (error) {
console.error('[%s] getDownloadStream: decipher stream error.', error);
callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
});
s3get.pipe(decrypt);
callback(null, decrypt);
});
});
}
function testConfig(apiConfig, callback) {