diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index cb21ef8a6..048d4d696 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -4129,6 +4129,11 @@ "from": "rndm@1.2.0", "resolved": "https://registry.npmjs.org/rndm/-/rndm-1.2.0.tgz" }, + "s3-block-read-stream": { + "version": "0.2.0", + "from": "s3-block-read-stream@latest", + "resolved": "https://registry.npmjs.org/s3-block-read-stream/-/s3-block-read-stream-0.2.0.tgz" + }, "safe-buffer": { "version": "5.0.1", "from": "safe-buffer@>=5.0.1 <6.0.0", diff --git a/package.json b/package.json index b513a16c6..66be3fd12 100644 --- a/package.json +++ b/package.json @@ -58,6 +58,7 @@ "password-generator": "^2.0.2", "progress-stream": "^2.0.0", "proxy-middleware": "^0.13.0", + "s3-block-read-stream": "^0.2.0", "safetydance": "^0.2.0", "semver": "^4.3.6", "showdown": "^1.6.0", diff --git a/src/storage/caas.js b/src/storage/caas.js index 6dd6b2c03..838697dda 100644 --- a/src/storage/caas.js +++ b/src/storage/caas.js @@ -19,6 +19,7 @@ var assert = require('assert'), once = require('once'), PassThrough = require('stream').PassThrough, path = require('path'), + S3BlockReadStream = require('s3-block-read-stream'), superagent = require('superagent'), targz = require('./targz.js'); @@ -119,9 +120,9 @@ function restore(apiConfig, backupId, destination, callback) { }; var s3 = new AWS.S3(credentials); - var s3get = s3.getObject(params).createReadStream(); + var multipartDownload = new S3BlockReadStream(s3, params, { blockSize: 64 * 1024 * 1024, logCallback: debug }); - s3get.on('error', function (error) { + multipartDownload.on('error', function (error) { // TODO ENOENT for the mock, fix upstream! if (error.code === 'NoSuchKey' || error.code === 'ENOENT') return callback(new BackupsError(BackupsError.NOT_FOUND)); @@ -129,7 +130,7 @@ function restore(apiConfig, backupId, destination, callback) { callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message)); }); - targz.extract(s3get, destination, apiConfig.key || null, callback); + targz.extract(multipartDownload, destination, apiConfig.key || null, callback); }); } diff --git a/src/storage/s3.js b/src/storage/s3.js index 49568f271..894216aa2 100644 --- a/src/storage/s3.js +++ b/src/storage/s3.js @@ -22,6 +22,7 @@ var assert = require('assert'), once = require('once'), PassThrough = require('stream').PassThrough, path = require('path'), + S3BlockReadStream = require('s3-block-read-stream'), targz = require('./targz.js'); // test only @@ -125,9 +126,9 @@ function restore(apiConfig, backupId, destination, callback) { var s3 = new AWS.S3(credentials); - var s3get = s3.getObject(params).createReadStream(); + var multipartDownload = new S3BlockReadStream(s3, params, { blockSize: 64 * 1024 * 1024, logCallback: debug }); - s3get.on('error', function (error) { + multipartDownload.on('error', function (error) { // TODO ENOENT for the mock, fix upstream! if (error.code === 'NoSuchKey' || error.code === 'ENOENT') return callback(new BackupsError(BackupsError.NOT_FOUND)); @@ -135,7 +136,7 @@ function restore(apiConfig, backupId, destination, callback) { callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message)); }); - targz.extract(s3get, destination, apiConfig.key || null, callback); + targz.extract(multipartDownload, destination, apiConfig.key || null, callback); }); }