Make copy() return event emitter

This way the storage logic does not need to rely on progress
This commit is contained in:
Girish Ramakrishnan
2017-10-04 11:00:30 -07:00
parent 38331e71e2
commit d70ff7cd5b
7 changed files with 49 additions and 30 deletions
+9 -5
View File
@@ -18,6 +18,7 @@ exports = module.exports = {
var assert = require('assert'),
BackupsError = require('../backups.js').BackupsError,
debug = require('debug')('box:storage/filesystem'),
EventEmitter = require('events'),
fs = require('fs'),
mkdirp = require('mkdirp'),
path = require('path'),
@@ -90,24 +91,27 @@ function downloadDir(apiConfig, backupFilePath, destDir, callback) {
});
}
function copy(apiConfig, oldFilePath, newFilePath, callback) {
function copy(apiConfig, oldFilePath, newFilePath) {
assert.strictEqual(typeof apiConfig, 'object');
assert.strictEqual(typeof oldFilePath, 'string');
assert.strictEqual(typeof newFilePath, 'string');
assert.strictEqual(typeof callback, 'function');
debug('copy: %s -> %s', oldFilePath, newFilePath);
var events = new EventEmitter();
mkdirp(path.dirname(newFilePath), function (error) {
if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message));
if (error) return events.emit('done', new BackupsError(BackupsError.EXTERNAL_ERROR, error.message));
// this will hardlink backups saving space
shell.exec('copy', '/bin/cp', [ '-al', oldFilePath, newFilePath ], { }, function (error) {
if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message));
if (error) return events.emit('done', new BackupsError(BackupsError.EXTERNAL_ERROR, error.message));
callback(null);
events.emit('done', null);
});
});
return events;
}
function remove(apiConfig, filename, callback) {
+6 -6
View File
@@ -20,7 +20,8 @@ exports = module.exports = {
testConfig: testConfig
};
var assert = require('assert');
var assert = require('assert'),
EventEmitter = require('events');
function upload(apiConfig, backupFilePath, sourceStream, callback) {
assert.strictEqual(typeof apiConfig, 'object');
@@ -51,15 +52,14 @@ function downloadDir(apiConfig, backupFilePath, destDir, callback) {
callback(new Error('not implemented'));
}
function copy(apiConfig, oldFilePath, newFilePath, callback) {
function copy(apiConfig, oldFilePath, newFilePath) {
assert.strictEqual(typeof apiConfig, 'object');
assert.strictEqual(typeof oldFilePath, 'string');
assert.strictEqual(typeof newFilePath, 'string');
assert.strictEqual(typeof callback, 'function');
// Result: none
callback(new Error('not implemented'));
var events = new EventEmitter();
process.nextTick(function () { events.emit('done', null); });
return events;
}
function remove(apiConfig, filename, callback) {
+6 -4
View File
@@ -15,7 +15,8 @@ exports = module.exports = {
};
var assert = require('assert'),
debug = require('debug')('box:storage/noop');
debug = require('debug')('box:storage/noop'),
EventEmitter = require('events');
function upload(apiConfig, backupFilePath, sourceStream, callback) {
assert.strictEqual(typeof apiConfig, 'object');
@@ -49,15 +50,16 @@ function downloadDir(apiConfig, backupFilePath, destDir, callback) {
callback(new Error('Cannot download from noop backend'));
}
function copy(apiConfig, oldFilePath, newFilePath, callback) {
function copy(apiConfig, oldFilePath, newFilePath) {
assert.strictEqual(typeof apiConfig, 'object');
assert.strictEqual(typeof oldFilePath, 'string');
assert.strictEqual(typeof newFilePath, 'string');
assert.strictEqual(typeof callback, 'function');
debug('copy: %s -> %s', oldFilePath, newFilePath);
callback(null);
var events = new EventEmitter();
process.nextTick(function () { events.emit('done', null); });
return events;
}
function remove(apiConfig, filename, callback) {
+14 -7
View File
@@ -24,12 +24,12 @@ var assert = require('assert'),
BackupsError = require('../backups.js').BackupsError,
config = require('../config.js'),
debug = require('debug')('box:storage/s3'),
EventEmitter = require('events'),
fs = require('fs'),
chunk = require('lodash.chunk'),
mkdirp = require('mkdirp'),
PassThrough = require('stream').PassThrough,
path = require('path'),
progress = require('../progress.js'),
S3BlockReadStream = require('s3-block-read-stream'),
superagent = require('superagent');
@@ -227,17 +227,16 @@ function downloadDir(apiConfig, backupFilePath, destDir, callback) {
}, callback);
}
function copy(apiConfig, oldFilePath, newFilePath, callback) {
function copy(apiConfig, oldFilePath, newFilePath) {
assert.strictEqual(typeof apiConfig, 'object');
assert.strictEqual(typeof oldFilePath, 'string');
assert.strictEqual(typeof newFilePath, 'string');
assert.strictEqual(typeof callback, 'function');
var events = new EventEmitter();
listDir(apiConfig, oldFilePath, { batchSize: 1 }, function copyFile(s3, content, iteratorCallback) {
var relativePath = path.relative(oldFilePath, content.Key);
progress.setDetail(progress.BACKUP, 'Copying ' + content.Key.slice(oldFilePath.length+1));
function done(error) {
if (error && error.code === 'NoSuchKey') return iteratorCallback(new BackupsError(BackupsError.NOT_FOUND, `Old backup not found: ${content.Key}`));
if (error) {
@@ -245,7 +244,7 @@ function copy(apiConfig, oldFilePath, newFilePath, callback) {
return iteratorCallback(new BackupsError(BackupsError.EXTERNAL_ERROR, `Error copying ${content.Key} : ${error.message}`));
}
iteratorCallback();
iteratorCallback(null);
}
var copyParams = {
@@ -255,10 +254,14 @@ function copy(apiConfig, oldFilePath, newFilePath, callback) {
// S3 copyObject has a file size limit of 5GB so if we have larger files, we do a multipart copy
if (content.Size < 5 * 1024 * 1024 * 1024) {
events.emit('progress', 'Copying ' + content.Key.slice(oldFilePath.length+1));
copyParams.CopySource = encodeURIComponent(path.join(apiConfig.bucket, content.Key)); // See aws-sdk-js/issues/1302
return s3.copyObject(copyParams, done);
}
events.emit('progress', 'Copying (multipart) ' + content.Key.slice(oldFilePath.length+1));
s3.createMultipartUpload(copyParams, function (error, result) {
if (error) return done(error);
@@ -307,7 +310,11 @@ function copy(apiConfig, oldFilePath, newFilePath, callback) {
copyNextChunk();
});
}, callback);
}, function (error) {
events.emit('done', error);
});
return events;
}
function remove(apiConfig, filename, callback) {