jjkiers/cloudron-box
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

read/write iv in the encrypted files

Browse Source 
part of #579
This commit is contained in:
Girish Ramakrishnan
2020-05-10 21:40:25 -07:00
parent 15d0275045
commit 56d3b38ce6
1 changed files with 115 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/backups.js
+115 -27
View File
@@ -69,6 +69,7 @@ var addons = require('./addons.js'),
syncer = require('./syncer.js'),
tar = require('tar-fs'),
tasks = require('./tasks.js'),
TransformStream = require('stream').Transform,
util = require('util'),
zlib = require('zlib');
@@ -194,12 +195,13 @@ function getBackupFilePath(backupConfig, backupId, format) {
function encryptFilePath(filePath, key) {
assert.strictEqual(typeof filePath, 'string');
assert.strictEqual(typeof key, 'string');
assert(Buffer.isBuffer(key));
var encryptedParts = filePath.split('/').map(function (part) {
const cipher = crypto.createCipher('aes-256-cbc', key);
let iv = crypto.createHmac('sha256', key).update(part).digest().slice(0, 16); // iv has to be deterministic, for our sync (copy) logic to work
const cipher = crypto.createCipheriv('aes-256-cbc', key, iv);
let crypt = cipher.update(part);
crypt = Buffer.concat([ crypt, cipher.final() ]);
crypt = Buffer.concat([ iv, crypt, cipher.final() ]);
return crypt.toString('base64') // ensures path is valid
.replace(/\//g, '-') // replace '/' of base64 since it conflicts with path separator
@@ -211,7 +213,7 @@ function encryptFilePath(filePath, key) {
function decryptFilePath(filePath, key) {
assert.strictEqual(typeof filePath, 'string');
assert.strictEqual(typeof key, 'string');
assert(Buffer.isBuffer(key));
let decryptedParts = [];
for (let part of filePath.split('/')) {
@@ -219,8 +221,10 @@ function decryptFilePath(filePath, key) {
part = part.replace(/-/g, '/'); // replace with '/'
try {
let decrypt = crypto.createDecipher('aes-256-cbc', key);
let text = decrypt.update(Buffer.from(part, 'base64'));
const buffer = Buffer.from(part, 'base64');
const iv = buffer.slice(0, 16);
let decrypt = crypto.createDecipheriv('aes-256-cbc', key, iv);
let text = decrypt.update(buffer.slice(16));
text = Buffer.concat([ text, decrypt.final() ]);
decryptedParts.push(text.toString('utf8'));
} catch (error) {
@@ -232,9 +236,76 @@ function decryptFilePath(filePath, key) {
return decryptedParts.join('/');
}
class EncryptStream extends TransformStream {
constructor(key) {
super();
this._ivPushed = false;
this._iv = crypto.randomBytes(16);
this._cipher = crypto.createCipheriv('aes-256-cbc', key, this._iv);
}
_transform(chunk, ignoredEncoding, callback) {
if (!this._ivPushed) {
this.push(this._iv);
this._ivPushed = true;
}
try {
const crypt = this._cipher.update(chunk);
callback(null, crypt);
} catch (error) {
callback(error);
}
}
_flush(callback) {
try {
const crypt = this._cipher.final();
callback(null, crypt);
} catch (error) {
callback(error);
}
}
}
class DecryptStream extends TransformStream {
constructor(key) {
super();
this._key = key;
this._iv = Buffer.alloc(0);
this._decipher = null;
}
_transform(chunk, ignoredEncoding, callback) {
if (this._iv.length !== 16) { // not gotten IV yet
const needed = 16 - this._iv.length;
this._iv = Buffer.concat([this._iv, chunk.slice(0, needed)]);
if (this._iv.length !== 16) return callback();
this._decipher = crypto.createDecipheriv('aes-256-cbc', this._key, this._iv);
chunk = chunk.slice(needed);
}
try {
const plainText = this._decipher.update(chunk);
callback(null, plainText);
} catch (error) {
callback(error);
}
}
_flush (callback) {
try {
const plainText = this._decipher.final();
callback(null, plainText);
} catch (error) {
callback(error);
}
}
}
function createReadStream(sourceFile, key) {
assert.strictEqual(typeof sourceFile, 'string');
assert(key === null || typeof key === 'string');
assert(key === null || Buffer.isBuffer(key));
var stream = fs.createReadStream(sourceFile);
var ps = progressStream({ time: 10000 }); // display a progress every 10 seconds
@@ -245,12 +316,14 @@ function createReadStream(sourceFile, key) {
});
if (key !== null) {
var encrypt = crypto.createCipher('aes-256-cbc', key);
encrypt.on('error', function (error) {
let encryptStream = new EncryptStream(key);
encryptStream.on('error', function (error) {
debug('createReadStream: encrypt stream error.', error);
ps.emit('error', new BoxError(BoxError.CRYPTO_ERROR, error.message));
});
return stream.pipe(encrypt).pipe(ps);
return stream.pipe(encryptStream).pipe(ps);
} else {
return stream.pipe(ps);
}
@@ -258,7 +331,7 @@ function createReadStream(sourceFile, key) {
function createWriteStream(destFile, key) {
assert.strictEqual(typeof destFile, 'string');
assert(key === null || typeof key === 'string');
assert(key === null || Buffer.isBuffer(key));
var stream = fs.createWriteStream(destFile);
var ps = progressStream({ time: 10000 }); // display a progress every 10 seconds
@@ -269,11 +342,12 @@ function createWriteStream(destFile, key) {
});
if (key !== null) {
var decrypt = crypto.createDecipher('aes-256-cbc', key);
let decrypt = new DecryptStream(key);
decrypt.on('error', function (error) {
debug('createWriteStream: decrypt stream error.', error);
ps.emit('error', new BoxError(BoxError.CRYPTO_ERROR, error.message));
});
ps.pipe(decrypt).pipe(stream);
} else {
ps.pipe(stream);
@@ -284,7 +358,7 @@ function createWriteStream(destFile, key) {
function tarPack(dataLayout, key, callback) {
assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
assert(key === null || typeof key === 'string');
assert(key === null || Buffer.isBuffer(key));
assert.strictEqual(typeof callback, 'function');
var pack = tar.pack('/', {
@@ -305,7 +379,7 @@ function tarPack(dataLayout, key, callback) {
});
var gzip = zlib.createGzip({});
var ps = progressStream({ time: 10000 }); // emit 'pgoress' every 10 seconds
var ps = progressStream({ time: 10000 }); // emit 'progress' every 10 seconds
pack.on('error', function (error) {
debug('tarPack: tar stream error.', error);
@@ -318,17 +392,26 @@ function tarPack(dataLayout, key, callback) {
});
if (key !== null) {
var encrypt = crypto.createCipher('aes-256-cbc', key);
encrypt.on('error', function (error) {
const encryptStream = new EncryptStream(key);
encryptStream.on('error', function (error) {
debug('tarPack: encrypt stream error.', error);
ps.emit('error', new BoxError(BoxError.EXTERNAL_ERROR, error.message));
});
pack.pipe(gzip).pipe(encrypt).pipe(ps);
pack.pipe(gzip).pipe(encryptStream).pipe(ps);
} else {
pack.pipe(gzip).pipe(ps);
}
callback(null, ps);
return callback(null, ps);
}
function aesKeyFromPassword(password) {
assert(password === null || typeof password === 'string');
if (!password) return null;
return crypto.scryptSync(password, Buffer.from('CLOUDRONSCRYPTSALT', 'utf8'), 32);
}
function sync(backupConfig, backupId, dataLayout, progressCallback, callback) {
@@ -340,11 +423,12 @@ function sync(backupConfig, backupId, dataLayout, progressCallback, callback) {
// the number here has to take into account the s3.upload partSize (which is 10MB). So 20=200MB
const concurrency = backupConfig.syncConcurrency || (backupConfig.provider === 's3' ? 20 : 10);
const aesKey = aesKeyFromPassword(backupConfig.key);
syncer.sync(dataLayout, function processTask(task, iteratorCallback) {
debug('sync: processing task: %j', task);
// the empty task.path is special to signify the directory
const destPath = task.path && backupConfig.key ? encryptFilePath(task.path, backupConfig.key) : task.path;
const destPath = task.path && backupConfig.key ? encryptFilePath(task.path, aesKey) : task.path;
const backupFilePath = path.join(getBackupFilePath(backupConfig, backupId, backupConfig.format), destPath);
if (task.operation === 'removedir') {
@@ -365,7 +449,7 @@ function sync(backupConfig, backupId, dataLayout, progressCallback, callback) {
if (task.operation === 'add') {
progressCallback({ message: `Adding ${task.path}` + (retryCount > 1 ? ` (Try ${retryCount})` : '') });
debug(`Adding ${task.path} position ${task.position} try ${retryCount}`);
var stream = createReadStream(dataLayout.toLocalPath('./' + task.path), backupConfig.key || null);
var stream = createReadStream(dataLayout.toLocalPath('./' + task.path), aesKey);
stream.on('error', function (error) {
debug(`read stream error for ${task.path}: ${error.message}`);
retryCallback();
@@ -463,10 +547,11 @@ function upload(backupId, format, dataLayoutString, progressCallback, callback)
if (error) return callback(error);
if (format === 'tgz') {
const aesKey = aesKeyFromPassword(backupConfig.key);
async.retry({ times: 5, interval: 20000 }, function (retryCallback) {
retryCallback = once(retryCallback); // protect again upload() erroring much later after tar stream error
tarPack(dataLayout, backupConfig.key || null, function (error, tarStream) {
tarPack(dataLayout, aesKey, function (error, tarStream) {
if (error) return retryCallback(error);
tarStream.on('progress', function (progress) {
@@ -492,7 +577,7 @@ function upload(backupId, format, dataLayoutString, progressCallback, callback)
function tarExtract(inStream, dataLayout, key, callback) {
assert.strictEqual(typeof inStream, 'object');
assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
assert(key === null || typeof key === 'string');
assert(key === null || Buffer.isBuffer(key));
assert.strictEqual(typeof callback, 'function');
var gunzip = zlib.createGunzip({});
@@ -528,7 +613,7 @@ function tarExtract(inStream, dataLayout, key, callback) {
});
if (key !== null) {
var decrypt = crypto.createDecipher('aes-256-cbc', key);
let decrypt = new DecryptStream(key);
decrypt.on('error', function (error) {
debug('tarExtract: decrypt stream error.', error);
emitError(new BoxError(BoxError.EXTERNAL_ERROR, `Failed to decrypt: ${error.message}`));
@@ -577,11 +662,13 @@ function downloadDir(backupConfig, backupFilePath, dataLayout, progressCallback,
debug(`downloadDir: ${backupFilePath} to ${dataLayout.toString()}`);
const aesKey = aesKeyFromPassword(backupConfig.key);
function downloadFile(entry, done) {
let relativePath = path.relative(backupFilePath, entry.fullPath);
if (backupConfig.key) {
relativePath = decryptFilePath(relativePath, backupConfig.key);
if (!relativePath) return done(new BoxError(BoxError.BAD_STATE, 'Unable to decrypt file'));
relativePath = decryptFilePath(relativePath, aesKey);
if (!relativePath) return done(new BoxError(BoxError.CRYPTO_ERROR, 'Unable to decrypt file'));
}
const destFilePath = dataLayout.toLocalPath('./' + relativePath);
@@ -589,7 +676,7 @@ function downloadDir(backupConfig, backupFilePath, dataLayout, progressCallback,
if (error) return done(new BoxError(BoxError.FS_ERROR, error.message));
async.retry({ times: 5, interval: 20000 }, function (retryCallback) {
let destStream = createWriteStream(destFilePath, backupConfig.key || null);
let destStream = createWriteStream(destFilePath, aesKey);
destStream.on('progress', function (progress) {
const transferred = Math.round(progress.transferred/1024/1024), speed = Math.round(progress.speed/1024/1024);
@@ -640,11 +727,12 @@ function download(backupConfig, backupId, format, dataLayout, progressCallback,
const backupFilePath = getBackupFilePath(backupConfig, backupId, format);
if (format === 'tgz') {
const aesKey = aesKeyFromPassword(backupConfig.key);
async.retry({ times: 5, interval: 20000 }, function (retryCallback) {
api(backupConfig.provider).download(backupConfig, backupFilePath, function (error, sourceStream) {
if (error) return retryCallback(error);
tarExtract(sourceStream, dataLayout, backupConfig.key || null, function (error, ps) {
tarExtract(sourceStream, dataLayout, aesKey, function (error, ps) {
if (error) return retryCallback(error);
ps.on('progress', function (progress) {