Make copy() return event emitter
This way the storage logic does not need to rely on progress
This commit is contained in:
@@ -18,6 +18,7 @@ exports = module.exports = {
|
||||
var assert = require('assert'),
|
||||
BackupsError = require('../backups.js').BackupsError,
|
||||
debug = require('debug')('box:storage/filesystem'),
|
||||
EventEmitter = require('events'),
|
||||
fs = require('fs'),
|
||||
mkdirp = require('mkdirp'),
|
||||
path = require('path'),
|
||||
@@ -90,24 +91,27 @@ function downloadDir(apiConfig, backupFilePath, destDir, callback) {
|
||||
});
|
||||
}
|
||||
|
||||
function copy(apiConfig, oldFilePath, newFilePath, callback) {
|
||||
function copy(apiConfig, oldFilePath, newFilePath) {
|
||||
assert.strictEqual(typeof apiConfig, 'object');
|
||||
assert.strictEqual(typeof oldFilePath, 'string');
|
||||
assert.strictEqual(typeof newFilePath, 'string');
|
||||
assert.strictEqual(typeof callback, 'function');
|
||||
|
||||
debug('copy: %s -> %s', oldFilePath, newFilePath);
|
||||
|
||||
var events = new EventEmitter();
|
||||
|
||||
mkdirp(path.dirname(newFilePath), function (error) {
|
||||
if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message));
|
||||
if (error) return events.emit('done', new BackupsError(BackupsError.EXTERNAL_ERROR, error.message));
|
||||
|
||||
// this will hardlink backups saving space
|
||||
shell.exec('copy', '/bin/cp', [ '-al', oldFilePath, newFilePath ], { }, function (error) {
|
||||
if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message));
|
||||
if (error) return events.emit('done', new BackupsError(BackupsError.EXTERNAL_ERROR, error.message));
|
||||
|
||||
callback(null);
|
||||
events.emit('done', null);
|
||||
});
|
||||
});
|
||||
|
||||
return events;
|
||||
}
|
||||
|
||||
function remove(apiConfig, filename, callback) {
|
||||
|
||||
@@ -20,7 +20,8 @@ exports = module.exports = {
|
||||
testConfig: testConfig
|
||||
};
|
||||
|
||||
var assert = require('assert');
|
||||
var assert = require('assert'),
|
||||
EventEmitter = require('events');
|
||||
|
||||
function upload(apiConfig, backupFilePath, sourceStream, callback) {
|
||||
assert.strictEqual(typeof apiConfig, 'object');
|
||||
@@ -51,15 +52,14 @@ function downloadDir(apiConfig, backupFilePath, destDir, callback) {
|
||||
callback(new Error('not implemented'));
|
||||
}
|
||||
|
||||
function copy(apiConfig, oldFilePath, newFilePath, callback) {
|
||||
function copy(apiConfig, oldFilePath, newFilePath) {
|
||||
assert.strictEqual(typeof apiConfig, 'object');
|
||||
assert.strictEqual(typeof oldFilePath, 'string');
|
||||
assert.strictEqual(typeof newFilePath, 'string');
|
||||
assert.strictEqual(typeof callback, 'function');
|
||||
|
||||
// Result: none
|
||||
|
||||
callback(new Error('not implemented'));
|
||||
var events = new EventEmitter();
|
||||
process.nextTick(function () { events.emit('done', null); });
|
||||
return events;
|
||||
}
|
||||
|
||||
function remove(apiConfig, filename, callback) {
|
||||
|
||||
+6
-4
@@ -15,7 +15,8 @@ exports = module.exports = {
|
||||
};
|
||||
|
||||
var assert = require('assert'),
|
||||
debug = require('debug')('box:storage/noop');
|
||||
debug = require('debug')('box:storage/noop'),
|
||||
EventEmitter = require('events');
|
||||
|
||||
function upload(apiConfig, backupFilePath, sourceStream, callback) {
|
||||
assert.strictEqual(typeof apiConfig, 'object');
|
||||
@@ -49,15 +50,16 @@ function downloadDir(apiConfig, backupFilePath, destDir, callback) {
|
||||
callback(new Error('Cannot download from noop backend'));
|
||||
}
|
||||
|
||||
function copy(apiConfig, oldFilePath, newFilePath, callback) {
|
||||
function copy(apiConfig, oldFilePath, newFilePath) {
|
||||
assert.strictEqual(typeof apiConfig, 'object');
|
||||
assert.strictEqual(typeof oldFilePath, 'string');
|
||||
assert.strictEqual(typeof newFilePath, 'string');
|
||||
assert.strictEqual(typeof callback, 'function');
|
||||
|
||||
debug('copy: %s -> %s', oldFilePath, newFilePath);
|
||||
|
||||
callback(null);
|
||||
var events = new EventEmitter();
|
||||
process.nextTick(function () { events.emit('done', null); });
|
||||
return events;
|
||||
}
|
||||
|
||||
function remove(apiConfig, filename, callback) {
|
||||
|
||||
+14
-7
@@ -24,12 +24,12 @@ var assert = require('assert'),
|
||||
BackupsError = require('../backups.js').BackupsError,
|
||||
config = require('../config.js'),
|
||||
debug = require('debug')('box:storage/s3'),
|
||||
EventEmitter = require('events'),
|
||||
fs = require('fs'),
|
||||
chunk = require('lodash.chunk'),
|
||||
mkdirp = require('mkdirp'),
|
||||
PassThrough = require('stream').PassThrough,
|
||||
path = require('path'),
|
||||
progress = require('../progress.js'),
|
||||
S3BlockReadStream = require('s3-block-read-stream'),
|
||||
superagent = require('superagent');
|
||||
|
||||
@@ -227,17 +227,16 @@ function downloadDir(apiConfig, backupFilePath, destDir, callback) {
|
||||
}, callback);
|
||||
}
|
||||
|
||||
function copy(apiConfig, oldFilePath, newFilePath, callback) {
|
||||
function copy(apiConfig, oldFilePath, newFilePath) {
|
||||
assert.strictEqual(typeof apiConfig, 'object');
|
||||
assert.strictEqual(typeof oldFilePath, 'string');
|
||||
assert.strictEqual(typeof newFilePath, 'string');
|
||||
assert.strictEqual(typeof callback, 'function');
|
||||
|
||||
var events = new EventEmitter();
|
||||
|
||||
listDir(apiConfig, oldFilePath, { batchSize: 1 }, function copyFile(s3, content, iteratorCallback) {
|
||||
var relativePath = path.relative(oldFilePath, content.Key);
|
||||
|
||||
progress.setDetail(progress.BACKUP, 'Copying ' + content.Key.slice(oldFilePath.length+1));
|
||||
|
||||
function done(error) {
|
||||
if (error && error.code === 'NoSuchKey') return iteratorCallback(new BackupsError(BackupsError.NOT_FOUND, `Old backup not found: ${content.Key}`));
|
||||
if (error) {
|
||||
@@ -245,7 +244,7 @@ function copy(apiConfig, oldFilePath, newFilePath, callback) {
|
||||
return iteratorCallback(new BackupsError(BackupsError.EXTERNAL_ERROR, `Error copying ${content.Key} : ${error.message}`));
|
||||
}
|
||||
|
||||
iteratorCallback();
|
||||
iteratorCallback(null);
|
||||
}
|
||||
|
||||
var copyParams = {
|
||||
@@ -255,10 +254,14 @@ function copy(apiConfig, oldFilePath, newFilePath, callback) {
|
||||
|
||||
// S3 copyObject has a file size limit of 5GB so if we have larger files, we do a multipart copy
|
||||
if (content.Size < 5 * 1024 * 1024 * 1024) {
|
||||
events.emit('progress', 'Copying ' + content.Key.slice(oldFilePath.length+1));
|
||||
|
||||
copyParams.CopySource = encodeURIComponent(path.join(apiConfig.bucket, content.Key)); // See aws-sdk-js/issues/1302
|
||||
return s3.copyObject(copyParams, done);
|
||||
}
|
||||
|
||||
events.emit('progress', 'Copying (multipart) ' + content.Key.slice(oldFilePath.length+1));
|
||||
|
||||
s3.createMultipartUpload(copyParams, function (error, result) {
|
||||
if (error) return done(error);
|
||||
|
||||
@@ -307,7 +310,11 @@ function copy(apiConfig, oldFilePath, newFilePath, callback) {
|
||||
|
||||
copyNextChunk();
|
||||
});
|
||||
}, callback);
|
||||
}, function (error) {
|
||||
events.emit('done', error);
|
||||
});
|
||||
|
||||
return events;
|
||||
}
|
||||
|
||||
function remove(apiConfig, filename, callback) {
|
||||
|
||||
Reference in New Issue
Block a user