jjkiers/cloudron-box
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Make copy() return event emitter

Browse Source 
This way the storage logic does not need to rely on progress
This commit is contained in:
Girish Ramakrishnan
2017-10-04 11:00:30 -07:00
parent 38331e71e2
commit d70ff7cd5b
7 changed files with 49 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/backups.js
+8 -2
View File
@@ -491,7 +491,10 @@ function rotateBoxBackup(backupConfig, timestamp, appBackupIds, callback) {
progress.setDetail(progress.BACKUP, 'Rotating box snapshot');
api(backupConfig.provider).copy(backupConfig, getBackupFilePath(backupConfig, 'snapshot/box', format), getBackupFilePath(backupConfig, backupId, format), function (copyBackupError) {
var copy = api(backupConfig.provider).copy(backupConfig, getBackupFilePath(backupConfig, 'snapshot/box', format), getBackupFilePath(backupConfig, backupId, format));
copy.on('progress', function (detail) { progress.setDetail(progress.BACKUP, detail); });
copy.on('done', function (copyBackupError) {
const state = copyBackupError ? backupdb.BACKUP_STATE_ERROR : backupdb.BACKUP_STATE_NORMAL;
backupdb.update(backupId, { state: state }, function (error) {
@@ -590,7 +593,10 @@ function rotateAppBackup(backupConfig, app, timestamp, callback) {
progress.setDetail(progress.BACKUP, 'Rotating app snapshot');
api(backupConfig.provider).copy(backupConfig, getBackupFilePath(backupConfig, `snapshot/app_${app.id}`, format), getBackupFilePath(backupConfig, backupId, format), function (copyBackupError) {
var copy = api(backupConfig.provider).copy(backupConfig, getBackupFilePath(backupConfig, `snapshot/app_${app.id}`, format), getBackupFilePath(backupConfig, backupId, format));
copy.on('progress', function (detail) { progress.setDetail(progress.BACKUP, detail); });
copy.on('done', function (copyBackupError) {
const state = copyBackupError ? backupdb.BACKUP_STATE_ERROR : backupdb.BACKUP_STATE_NORMAL;
debugApp(app, 'rotateAppBackup: successful id:%s', backupId);
src/progress.js
-3
View File
@@ -12,7 +12,6 @@ exports = module.exports = {
};
var assert = require('assert'),
config = require('./config.js'),
debug = require('debug')('box:progress');
// if progress.update or progress.backup are object, they will contain 'percent' and 'message' properties
@@ -42,8 +41,6 @@ function setDetail(tag, detail) {
assert.strictEqual(typeof tag, 'string');
assert.strictEqual(typeof detail, 'string');
if (config.TEST && !progress[tag]) progress[tag] = { };
progress[tag].detail = detail;
}
src/storage/filesystem.js
+9 -5
View File
@@ -18,6 +18,7 @@ exports = module.exports = {
var assert = require('assert'),
BackupsError = require('../backups.js').BackupsError,
debug = require('debug')('box:storage/filesystem'),
EventEmitter = require('events'),
fs = require('fs'),
mkdirp = require('mkdirp'),
path = require('path'),
@@ -90,24 +91,27 @@ function downloadDir(apiConfig, backupFilePath, destDir, callback) {
});
}
function copy(apiConfig, oldFilePath, newFilePath, callback) {
function copy(apiConfig, oldFilePath, newFilePath) {
assert.strictEqual(typeof apiConfig, 'object');
assert.strictEqual(typeof oldFilePath, 'string');
assert.strictEqual(typeof newFilePath, 'string');
assert.strictEqual(typeof callback, 'function');
debug('copy: %s -> %s', oldFilePath, newFilePath);
var events = new EventEmitter();
mkdirp(path.dirname(newFilePath), function (error) {
if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message));
if (error) return events.emit('done', new BackupsError(BackupsError.EXTERNAL_ERROR, error.message));
// this will hardlink backups saving space
shell.exec('copy', '/bin/cp', [ '-al', oldFilePath, newFilePath ], { }, function (error) {
if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message));
if (error) return events.emit('done', new BackupsError(BackupsError.EXTERNAL_ERROR, error.message));
callback(null);
events.emit('done', null);
});
});
return events;
}
function remove(apiConfig, filename, callback) {
src/storage/interface.js
+6 -6
View File
@@ -20,7 +20,8 @@ exports = module.exports = {
testConfig: testConfig
};
var assert = require('assert');
var assert = require('assert'),
EventEmitter = require('events');
function upload(apiConfig, backupFilePath, sourceStream, callback) {
assert.strictEqual(typeof apiConfig, 'object');
@@ -51,15 +52,14 @@ function downloadDir(apiConfig, backupFilePath, destDir, callback) {
callback(new Error('not implemented'));
}
function copy(apiConfig, oldFilePath, newFilePath, callback) {
function copy(apiConfig, oldFilePath, newFilePath) {
assert.strictEqual(typeof apiConfig, 'object');
assert.strictEqual(typeof oldFilePath, 'string');
assert.strictEqual(typeof newFilePath, 'string');
assert.strictEqual(typeof callback, 'function');
// Result: none
callback(new Error('not implemented'));
var events = new EventEmitter();
process.nextTick(function () { events.emit('done', null); });
return events;
}
function remove(apiConfig, filename, callback) {
src/storage/noop.js
+6 -4
View File
@@ -15,7 +15,8 @@ exports = module.exports = {
};
var assert = require('assert'),
debug = require('debug')('box:storage/noop');
debug = require('debug')('box:storage/noop'),
EventEmitter = require('events');
function upload(apiConfig, backupFilePath, sourceStream, callback) {
assert.strictEqual(typeof apiConfig, 'object');
@@ -49,15 +50,16 @@ function downloadDir(apiConfig, backupFilePath, destDir, callback) {
callback(new Error('Cannot download from noop backend'));
}
function copy(apiConfig, oldFilePath, newFilePath, callback) {
function copy(apiConfig, oldFilePath, newFilePath) {
assert.strictEqual(typeof apiConfig, 'object');
assert.strictEqual(typeof oldFilePath, 'string');
assert.strictEqual(typeof newFilePath, 'string');
assert.strictEqual(typeof callback, 'function');
debug('copy: %s -> %s', oldFilePath, newFilePath);
callback(null);
var events = new EventEmitter();
process.nextTick(function () { events.emit('done', null); });
return events;
}
function remove(apiConfig, filename, callback) {
src/storage/s3.js
+14 -7
View File
@@ -24,12 +24,12 @@ var assert = require('assert'),
BackupsError = require('../backups.js').BackupsError,
config = require('../config.js'),
debug = require('debug')('box:storage/s3'),
EventEmitter = require('events'),
fs = require('fs'),
chunk = require('lodash.chunk'),
mkdirp = require('mkdirp'),
PassThrough = require('stream').PassThrough,
path = require('path'),
progress = require('../progress.js'),
S3BlockReadStream = require('s3-block-read-stream'),
superagent = require('superagent');
@@ -227,17 +227,16 @@ function downloadDir(apiConfig, backupFilePath, destDir, callback) {
}, callback);
}
function copy(apiConfig, oldFilePath, newFilePath, callback) {
function copy(apiConfig, oldFilePath, newFilePath) {
assert.strictEqual(typeof apiConfig, 'object');
assert.strictEqual(typeof oldFilePath, 'string');
assert.strictEqual(typeof newFilePath, 'string');
assert.strictEqual(typeof callback, 'function');
var events = new EventEmitter();
listDir(apiConfig, oldFilePath, { batchSize: 1 }, function copyFile(s3, content, iteratorCallback) {
var relativePath = path.relative(oldFilePath, content.Key);
progress.setDetail(progress.BACKUP, 'Copying ' + content.Key.slice(oldFilePath.length+1));
function done(error) {
if (error && error.code === 'NoSuchKey') return iteratorCallback(new BackupsError(BackupsError.NOT_FOUND, `Old backup not found: ${content.Key}`));
if (error) {
@@ -245,7 +244,7 @@ function copy(apiConfig, oldFilePath, newFilePath, callback) {
return iteratorCallback(new BackupsError(BackupsError.EXTERNAL_ERROR, `Error copying ${content.Key} : ${error.message}`));
}
iteratorCallback();
iteratorCallback(null);
}
var copyParams = {
@@ -255,10 +254,14 @@ function copy(apiConfig, oldFilePath, newFilePath, callback) {
// S3 copyObject has a file size limit of 5GB so if we have larger files, we do a multipart copy
if (content.Size < 5 * 1024 * 1024 * 1024) {
events.emit('progress', 'Copying ' + content.Key.slice(oldFilePath.length+1));
copyParams.CopySource = encodeURIComponent(path.join(apiConfig.bucket, content.Key)); // See aws-sdk-js/issues/1302
return s3.copyObject(copyParams, done);
}
events.emit('progress', 'Copying (multipart) ' + content.Key.slice(oldFilePath.length+1));
s3.createMultipartUpload(copyParams, function (error, result) {
if (error) return done(error);
@@ -307,7 +310,11 @@ function copy(apiConfig, oldFilePath, newFilePath, callback) {
copyNextChunk();
});
}, callback);
}, function (error) {
events.emit('done', error);
});
return events;
}
function remove(apiConfig, filename, callback) {
src/test/storage-test.js
+6 -3
View File
@@ -112,7 +112,8 @@ describe('Storage', function () {
var sourceFile = gTmpFolder + '/uploadtest/test.txt'; // keep the test within save device
var destFile = gTmpFolder + '/uploadtest/test-hardlink.txt';
filesystem.copy(gBackupConfig, sourceFile, destFile, function (error) {
var events = filesystem.copy(gBackupConfig, sourceFile, destFile);
events.on('done', function (error) {
expect(error).to.be(null);
expect(fs.statSync(destFile).nlink).to.be(2); // created a hardlink
done();
@@ -169,7 +170,8 @@ describe('Storage', function () {
});
it('can copy', function (done) {
noop.copy(gBackupConfig, 'sourceFile', 'destFile', function (error) {
var events = noop.copy(gBackupConfig, 'sourceFile', 'destFile');
events.on('done', function (error) {
expect(error).to.be(null);
done();
});
@@ -256,7 +258,8 @@ describe('Storage', function () {
var sourceKey = 'uploadtest';
s3.copy(gBackupConfig, sourceKey, 'uploadtest-copy', function (error) {
var events = s3.copy(gBackupConfig, sourceKey, 'uploadtest-copy');
events.on('done', function (error) {
var sourceFile = path.join(__dirname, 'storage/data/test.txt');
expect(error).to.be(null);
expect(fs.statSync(path.join(gS3Folder, 'uploadtest-copy/test.txt')).size).to.be(fs.statSync(sourceFile).size);