diff --git a/src/backups.js b/src/backups.js index 28bc072ea..ce63bd66e 100644 --- a/src/backups.js +++ b/src/backups.js @@ -69,6 +69,7 @@ var addons = require('./addons.js'), syncer = require('./syncer.js'), tar = require('tar-fs'), tasks = require('./tasks.js'), + TransformStream = require('stream').Transform, util = require('util'), zlib = require('zlib'); @@ -194,12 +195,13 @@ function getBackupFilePath(backupConfig, backupId, format) { function encryptFilePath(filePath, key) { assert.strictEqual(typeof filePath, 'string'); - assert.strictEqual(typeof key, 'string'); + assert(Buffer.isBuffer(key)); var encryptedParts = filePath.split('/').map(function (part) { - const cipher = crypto.createCipher('aes-256-cbc', key); + let iv = crypto.createHmac('sha256', key).update(part).digest().slice(0, 16); // iv has to be deterministic, for our sync (copy) logic to work + const cipher = crypto.createCipheriv('aes-256-cbc', key, iv); let crypt = cipher.update(part); - crypt = Buffer.concat([ crypt, cipher.final() ]); + crypt = Buffer.concat([ iv, crypt, cipher.final() ]); return crypt.toString('base64') // ensures path is valid .replace(/\//g, '-') // replace '/' of base64 since it conflicts with path separator @@ -211,7 +213,7 @@ function encryptFilePath(filePath, key) { function decryptFilePath(filePath, key) { assert.strictEqual(typeof filePath, 'string'); - assert.strictEqual(typeof key, 'string'); + assert(Buffer.isBuffer(key)); let decryptedParts = []; for (let part of filePath.split('/')) { @@ -219,8 +221,10 @@ function decryptFilePath(filePath, key) { part = part.replace(/-/g, '/'); // replace with '/' try { - let decrypt = crypto.createDecipher('aes-256-cbc', key); - let text = decrypt.update(Buffer.from(part, 'base64')); + const buffer = Buffer.from(part, 'base64'); + const iv = buffer.slice(0, 16); + let decrypt = crypto.createDecipheriv('aes-256-cbc', key, iv); + let text = decrypt.update(buffer.slice(16)); text = Buffer.concat([ text, decrypt.final() ]); decryptedParts.push(text.toString('utf8')); } catch (error) { @@ -232,9 +236,76 @@ function decryptFilePath(filePath, key) { return decryptedParts.join('/'); } +class EncryptStream extends TransformStream { + constructor(key) { + super(); + this._ivPushed = false; + this._iv = crypto.randomBytes(16); + this._cipher = crypto.createCipheriv('aes-256-cbc', key, this._iv); + } + + _transform(chunk, ignoredEncoding, callback) { + if (!this._ivPushed) { + this.push(this._iv); + this._ivPushed = true; + } + try { + const crypt = this._cipher.update(chunk); + callback(null, crypt); + } catch (error) { + callback(error); + } + } + + _flush(callback) { + try { + const crypt = this._cipher.final(); + callback(null, crypt); + } catch (error) { + callback(error); + } + } +} + +class DecryptStream extends TransformStream { + constructor(key) { + super(); + this._key = key; + this._iv = Buffer.alloc(0); + this._decipher = null; + } + + _transform(chunk, ignoredEncoding, callback) { + if (this._iv.length !== 16) { // not gotten IV yet + const needed = 16 - this._iv.length; + this._iv = Buffer.concat([this._iv, chunk.slice(0, needed)]); + if (this._iv.length !== 16) return callback(); + + this._decipher = crypto.createDecipheriv('aes-256-cbc', this._key, this._iv); + chunk = chunk.slice(needed); + } + + try { + const plainText = this._decipher.update(chunk); + callback(null, plainText); + } catch (error) { + callback(error); + } + } + + _flush (callback) { + try { + const plainText = this._decipher.final(); + callback(null, plainText); + } catch (error) { + callback(error); + } + } +} + function createReadStream(sourceFile, key) { assert.strictEqual(typeof sourceFile, 'string'); - assert(key === null || typeof key === 'string'); + assert(key === null || Buffer.isBuffer(key)); var stream = fs.createReadStream(sourceFile); var ps = progressStream({ time: 10000 }); // display a progress every 10 seconds @@ -245,12 +316,14 @@ function createReadStream(sourceFile, key) { }); if (key !== null) { - var encrypt = crypto.createCipher('aes-256-cbc', key); - encrypt.on('error', function (error) { + let encryptStream = new EncryptStream(key); + + encryptStream.on('error', function (error) { debug('createReadStream: encrypt stream error.', error); ps.emit('error', new BoxError(BoxError.CRYPTO_ERROR, error.message)); }); - return stream.pipe(encrypt).pipe(ps); + + return stream.pipe(encryptStream).pipe(ps); } else { return stream.pipe(ps); } @@ -258,7 +331,7 @@ function createReadStream(sourceFile, key) { function createWriteStream(destFile, key) { assert.strictEqual(typeof destFile, 'string'); - assert(key === null || typeof key === 'string'); + assert(key === null || Buffer.isBuffer(key)); var stream = fs.createWriteStream(destFile); var ps = progressStream({ time: 10000 }); // display a progress every 10 seconds @@ -269,11 +342,12 @@ function createWriteStream(destFile, key) { }); if (key !== null) { - var decrypt = crypto.createDecipher('aes-256-cbc', key); + let decrypt = new DecryptStream(key); decrypt.on('error', function (error) { debug('createWriteStream: decrypt stream error.', error); ps.emit('error', new BoxError(BoxError.CRYPTO_ERROR, error.message)); }); + ps.pipe(decrypt).pipe(stream); } else { ps.pipe(stream); @@ -284,7 +358,7 @@ function createWriteStream(destFile, key) { function tarPack(dataLayout, key, callback) { assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout'); - assert(key === null || typeof key === 'string'); + assert(key === null || Buffer.isBuffer(key)); assert.strictEqual(typeof callback, 'function'); var pack = tar.pack('/', { @@ -305,7 +379,7 @@ function tarPack(dataLayout, key, callback) { }); var gzip = zlib.createGzip({}); - var ps = progressStream({ time: 10000 }); // emit 'pgoress' every 10 seconds + var ps = progressStream({ time: 10000 }); // emit 'progress' every 10 seconds pack.on('error', function (error) { debug('tarPack: tar stream error.', error); @@ -318,17 +392,26 @@ function tarPack(dataLayout, key, callback) { }); if (key !== null) { - var encrypt = crypto.createCipher('aes-256-cbc', key); - encrypt.on('error', function (error) { + const encryptStream = new EncryptStream(key); + encryptStream.on('error', function (error) { debug('tarPack: encrypt stream error.', error); ps.emit('error', new BoxError(BoxError.EXTERNAL_ERROR, error.message)); }); - pack.pipe(gzip).pipe(encrypt).pipe(ps); + + pack.pipe(gzip).pipe(encryptStream).pipe(ps); } else { pack.pipe(gzip).pipe(ps); } - callback(null, ps); + return callback(null, ps); +} + +function aesKeyFromPassword(password) { + assert(password === null || typeof password === 'string'); + + if (!password) return null; + + return crypto.scryptSync(password, Buffer.from('CLOUDRONSCRYPTSALT', 'utf8'), 32); } function sync(backupConfig, backupId, dataLayout, progressCallback, callback) { @@ -340,11 +423,12 @@ function sync(backupConfig, backupId, dataLayout, progressCallback, callback) { // the number here has to take into account the s3.upload partSize (which is 10MB). So 20=200MB const concurrency = backupConfig.syncConcurrency || (backupConfig.provider === 's3' ? 20 : 10); + const aesKey = aesKeyFromPassword(backupConfig.key); syncer.sync(dataLayout, function processTask(task, iteratorCallback) { debug('sync: processing task: %j', task); // the empty task.path is special to signify the directory - const destPath = task.path && backupConfig.key ? encryptFilePath(task.path, backupConfig.key) : task.path; + const destPath = task.path && backupConfig.key ? encryptFilePath(task.path, aesKey) : task.path; const backupFilePath = path.join(getBackupFilePath(backupConfig, backupId, backupConfig.format), destPath); if (task.operation === 'removedir') { @@ -365,7 +449,7 @@ function sync(backupConfig, backupId, dataLayout, progressCallback, callback) { if (task.operation === 'add') { progressCallback({ message: `Adding ${task.path}` + (retryCount > 1 ? ` (Try ${retryCount})` : '') }); debug(`Adding ${task.path} position ${task.position} try ${retryCount}`); - var stream = createReadStream(dataLayout.toLocalPath('./' + task.path), backupConfig.key || null); + var stream = createReadStream(dataLayout.toLocalPath('./' + task.path), aesKey); stream.on('error', function (error) { debug(`read stream error for ${task.path}: ${error.message}`); retryCallback(); @@ -463,10 +547,11 @@ function upload(backupId, format, dataLayoutString, progressCallback, callback) if (error) return callback(error); if (format === 'tgz') { + const aesKey = aesKeyFromPassword(backupConfig.key); async.retry({ times: 5, interval: 20000 }, function (retryCallback) { retryCallback = once(retryCallback); // protect again upload() erroring much later after tar stream error - tarPack(dataLayout, backupConfig.key || null, function (error, tarStream) { + tarPack(dataLayout, aesKey, function (error, tarStream) { if (error) return retryCallback(error); tarStream.on('progress', function (progress) { @@ -492,7 +577,7 @@ function upload(backupId, format, dataLayoutString, progressCallback, callback) function tarExtract(inStream, dataLayout, key, callback) { assert.strictEqual(typeof inStream, 'object'); assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout'); - assert(key === null || typeof key === 'string'); + assert(key === null || Buffer.isBuffer(key)); assert.strictEqual(typeof callback, 'function'); var gunzip = zlib.createGunzip({}); @@ -528,7 +613,7 @@ function tarExtract(inStream, dataLayout, key, callback) { }); if (key !== null) { - var decrypt = crypto.createDecipher('aes-256-cbc', key); + let decrypt = new DecryptStream(key); decrypt.on('error', function (error) { debug('tarExtract: decrypt stream error.', error); emitError(new BoxError(BoxError.EXTERNAL_ERROR, `Failed to decrypt: ${error.message}`)); @@ -577,11 +662,13 @@ function downloadDir(backupConfig, backupFilePath, dataLayout, progressCallback, debug(`downloadDir: ${backupFilePath} to ${dataLayout.toString()}`); + const aesKey = aesKeyFromPassword(backupConfig.key); + function downloadFile(entry, done) { let relativePath = path.relative(backupFilePath, entry.fullPath); if (backupConfig.key) { - relativePath = decryptFilePath(relativePath, backupConfig.key); - if (!relativePath) return done(new BoxError(BoxError.BAD_STATE, 'Unable to decrypt file')); + relativePath = decryptFilePath(relativePath, aesKey); + if (!relativePath) return done(new BoxError(BoxError.CRYPTO_ERROR, 'Unable to decrypt file')); } const destFilePath = dataLayout.toLocalPath('./' + relativePath); @@ -589,7 +676,7 @@ function downloadDir(backupConfig, backupFilePath, dataLayout, progressCallback, if (error) return done(new BoxError(BoxError.FS_ERROR, error.message)); async.retry({ times: 5, interval: 20000 }, function (retryCallback) { - let destStream = createWriteStream(destFilePath, backupConfig.key || null); + let destStream = createWriteStream(destFilePath, aesKey); destStream.on('progress', function (progress) { const transferred = Math.round(progress.transferred/1024/1024), speed = Math.round(progress.speed/1024/1024); @@ -640,11 +727,12 @@ function download(backupConfig, backupId, format, dataLayout, progressCallback, const backupFilePath = getBackupFilePath(backupConfig, backupId, format); if (format === 'tgz') { + const aesKey = aesKeyFromPassword(backupConfig.key); async.retry({ times: 5, interval: 20000 }, function (retryCallback) { api(backupConfig.provider).download(backupConfig, backupFilePath, function (error, sourceStream) { if (error) return retryCallback(error); - tarExtract(sourceStream, dataLayout, backupConfig.key || null, function (error, ps) { + tarExtract(sourceStream, dataLayout, aesKey, function (error, ps) { if (error) return retryCallback(error); ps.on('progress', function (progress) {