diff --git a/src/backups.js b/src/backups.js index 269fb664f..4f8afcc32 100644 --- a/src/backups.js +++ b/src/backups.js @@ -491,7 +491,10 @@ function rotateBoxBackup(backupConfig, timestamp, appBackupIds, callback) { progress.setDetail(progress.BACKUP, 'Rotating box snapshot'); - api(backupConfig.provider).copy(backupConfig, getBackupFilePath(backupConfig, 'snapshot/box', format), getBackupFilePath(backupConfig, backupId, format), function (copyBackupError) { + var copy = api(backupConfig.provider).copy(backupConfig, getBackupFilePath(backupConfig, 'snapshot/box', format), getBackupFilePath(backupConfig, backupId, format)); + copy.on('progress', function (detail) { progress.setDetail(progress.BACKUP, detail); }); + + copy.on('done', function (copyBackupError) { const state = copyBackupError ? backupdb.BACKUP_STATE_ERROR : backupdb.BACKUP_STATE_NORMAL; backupdb.update(backupId, { state: state }, function (error) { @@ -590,7 +593,10 @@ function rotateAppBackup(backupConfig, app, timestamp, callback) { progress.setDetail(progress.BACKUP, 'Rotating app snapshot'); - api(backupConfig.provider).copy(backupConfig, getBackupFilePath(backupConfig, `snapshot/app_${app.id}`, format), getBackupFilePath(backupConfig, backupId, format), function (copyBackupError) { + var copy = api(backupConfig.provider).copy(backupConfig, getBackupFilePath(backupConfig, `snapshot/app_${app.id}`, format), getBackupFilePath(backupConfig, backupId, format)); + copy.on('progress', function (detail) { progress.setDetail(progress.BACKUP, detail); }); + + copy.on('done', function (copyBackupError) { const state = copyBackupError ? backupdb.BACKUP_STATE_ERROR : backupdb.BACKUP_STATE_NORMAL; debugApp(app, 'rotateAppBackup: successful id:%s', backupId); diff --git a/src/progress.js b/src/progress.js index 203268192..62416145f 100644 --- a/src/progress.js +++ b/src/progress.js @@ -12,7 +12,6 @@ exports = module.exports = { }; var assert = require('assert'), - config = require('./config.js'), debug = require('debug')('box:progress'); // if progress.update or progress.backup are object, they will contain 'percent' and 'message' properties @@ -42,8 +41,6 @@ function setDetail(tag, detail) { assert.strictEqual(typeof tag, 'string'); assert.strictEqual(typeof detail, 'string'); - if (config.TEST && !progress[tag]) progress[tag] = { }; - progress[tag].detail = detail; } diff --git a/src/storage/filesystem.js b/src/storage/filesystem.js index ac3580fc0..9e0e87648 100644 --- a/src/storage/filesystem.js +++ b/src/storage/filesystem.js @@ -18,6 +18,7 @@ exports = module.exports = { var assert = require('assert'), BackupsError = require('../backups.js').BackupsError, debug = require('debug')('box:storage/filesystem'), + EventEmitter = require('events'), fs = require('fs'), mkdirp = require('mkdirp'), path = require('path'), @@ -90,24 +91,27 @@ function downloadDir(apiConfig, backupFilePath, destDir, callback) { }); } -function copy(apiConfig, oldFilePath, newFilePath, callback) { +function copy(apiConfig, oldFilePath, newFilePath) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof oldFilePath, 'string'); assert.strictEqual(typeof newFilePath, 'string'); - assert.strictEqual(typeof callback, 'function'); debug('copy: %s -> %s', oldFilePath, newFilePath); + var events = new EventEmitter(); + mkdirp(path.dirname(newFilePath), function (error) { - if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message)); + if (error) return events.emit('done', new BackupsError(BackupsError.EXTERNAL_ERROR, error.message)); // this will hardlink backups saving space shell.exec('copy', '/bin/cp', [ '-al', oldFilePath, newFilePath ], { }, function (error) { - if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message)); + if (error) return events.emit('done', new BackupsError(BackupsError.EXTERNAL_ERROR, error.message)); - callback(null); + events.emit('done', null); }); }); + + return events; } function remove(apiConfig, filename, callback) { diff --git a/src/storage/interface.js b/src/storage/interface.js index 0be9390e9..2e9032788 100644 --- a/src/storage/interface.js +++ b/src/storage/interface.js @@ -20,7 +20,8 @@ exports = module.exports = { testConfig: testConfig }; -var assert = require('assert'); +var assert = require('assert'), + EventEmitter = require('events'); function upload(apiConfig, backupFilePath, sourceStream, callback) { assert.strictEqual(typeof apiConfig, 'object'); @@ -51,15 +52,14 @@ function downloadDir(apiConfig, backupFilePath, destDir, callback) { callback(new Error('not implemented')); } -function copy(apiConfig, oldFilePath, newFilePath, callback) { +function copy(apiConfig, oldFilePath, newFilePath) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof oldFilePath, 'string'); assert.strictEqual(typeof newFilePath, 'string'); - assert.strictEqual(typeof callback, 'function'); - // Result: none - - callback(new Error('not implemented')); + var events = new EventEmitter(); + process.nextTick(function () { events.emit('done', null); }); + return events; } function remove(apiConfig, filename, callback) { diff --git a/src/storage/noop.js b/src/storage/noop.js index a67923e7f..ea0c9b277 100644 --- a/src/storage/noop.js +++ b/src/storage/noop.js @@ -15,7 +15,8 @@ exports = module.exports = { }; var assert = require('assert'), - debug = require('debug')('box:storage/noop'); + debug = require('debug')('box:storage/noop'), + EventEmitter = require('events'); function upload(apiConfig, backupFilePath, sourceStream, callback) { assert.strictEqual(typeof apiConfig, 'object'); @@ -49,15 +50,16 @@ function downloadDir(apiConfig, backupFilePath, destDir, callback) { callback(new Error('Cannot download from noop backend')); } -function copy(apiConfig, oldFilePath, newFilePath, callback) { +function copy(apiConfig, oldFilePath, newFilePath) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof oldFilePath, 'string'); assert.strictEqual(typeof newFilePath, 'string'); - assert.strictEqual(typeof callback, 'function'); debug('copy: %s -> %s', oldFilePath, newFilePath); - callback(null); + var events = new EventEmitter(); + process.nextTick(function () { events.emit('done', null); }); + return events; } function remove(apiConfig, filename, callback) { diff --git a/src/storage/s3.js b/src/storage/s3.js index 43f2f6845..39b739d4b 100644 --- a/src/storage/s3.js +++ b/src/storage/s3.js @@ -24,12 +24,12 @@ var assert = require('assert'), BackupsError = require('../backups.js').BackupsError, config = require('../config.js'), debug = require('debug')('box:storage/s3'), + EventEmitter = require('events'), fs = require('fs'), chunk = require('lodash.chunk'), mkdirp = require('mkdirp'), PassThrough = require('stream').PassThrough, path = require('path'), - progress = require('../progress.js'), S3BlockReadStream = require('s3-block-read-stream'), superagent = require('superagent'); @@ -227,17 +227,16 @@ function downloadDir(apiConfig, backupFilePath, destDir, callback) { }, callback); } -function copy(apiConfig, oldFilePath, newFilePath, callback) { +function copy(apiConfig, oldFilePath, newFilePath) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof oldFilePath, 'string'); assert.strictEqual(typeof newFilePath, 'string'); - assert.strictEqual(typeof callback, 'function'); + + var events = new EventEmitter(); listDir(apiConfig, oldFilePath, { batchSize: 1 }, function copyFile(s3, content, iteratorCallback) { var relativePath = path.relative(oldFilePath, content.Key); - progress.setDetail(progress.BACKUP, 'Copying ' + content.Key.slice(oldFilePath.length+1)); - function done(error) { if (error && error.code === 'NoSuchKey') return iteratorCallback(new BackupsError(BackupsError.NOT_FOUND, `Old backup not found: ${content.Key}`)); if (error) { @@ -245,7 +244,7 @@ function copy(apiConfig, oldFilePath, newFilePath, callback) { return iteratorCallback(new BackupsError(BackupsError.EXTERNAL_ERROR, `Error copying ${content.Key} : ${error.message}`)); } - iteratorCallback(); + iteratorCallback(null); } var copyParams = { @@ -255,10 +254,14 @@ function copy(apiConfig, oldFilePath, newFilePath, callback) { // S3 copyObject has a file size limit of 5GB so if we have larger files, we do a multipart copy if (content.Size < 5 * 1024 * 1024 * 1024) { + events.emit('progress', 'Copying ' + content.Key.slice(oldFilePath.length+1)); + copyParams.CopySource = encodeURIComponent(path.join(apiConfig.bucket, content.Key)); // See aws-sdk-js/issues/1302 return s3.copyObject(copyParams, done); } + events.emit('progress', 'Copying (multipart) ' + content.Key.slice(oldFilePath.length+1)); + s3.createMultipartUpload(copyParams, function (error, result) { if (error) return done(error); @@ -307,7 +310,11 @@ function copy(apiConfig, oldFilePath, newFilePath, callback) { copyNextChunk(); }); - }, callback); + }, function (error) { + events.emit('done', error); + }); + + return events; } function remove(apiConfig, filename, callback) { diff --git a/src/test/storage-test.js b/src/test/storage-test.js index df3b15797..094b98a99 100644 --- a/src/test/storage-test.js +++ b/src/test/storage-test.js @@ -112,7 +112,8 @@ describe('Storage', function () { var sourceFile = gTmpFolder + '/uploadtest/test.txt'; // keep the test within save device var destFile = gTmpFolder + '/uploadtest/test-hardlink.txt'; - filesystem.copy(gBackupConfig, sourceFile, destFile, function (error) { + var events = filesystem.copy(gBackupConfig, sourceFile, destFile); + events.on('done', function (error) { expect(error).to.be(null); expect(fs.statSync(destFile).nlink).to.be(2); // created a hardlink done(); @@ -169,7 +170,8 @@ describe('Storage', function () { }); it('can copy', function (done) { - noop.copy(gBackupConfig, 'sourceFile', 'destFile', function (error) { + var events = noop.copy(gBackupConfig, 'sourceFile', 'destFile'); + events.on('done', function (error) { expect(error).to.be(null); done(); }); @@ -256,7 +258,8 @@ describe('Storage', function () { var sourceKey = 'uploadtest'; - s3.copy(gBackupConfig, sourceKey, 'uploadtest-copy', function (error) { + var events = s3.copy(gBackupConfig, sourceKey, 'uploadtest-copy'); + events.on('done', function (error) { var sourceFile = path.join(__dirname, 'storage/data/test.txt'); expect(error).to.be(null); expect(fs.statSync(path.join(gS3Folder, 'uploadtest-copy/test.txt')).size).to.be(fs.statSync(sourceFile).size);