diff --git a/package-lock.json b/package-lock.json index 8ed07d55c..ed846a2b8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1171,8 +1171,7 @@ "balanced-match": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", - "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", - "dev": true + "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" }, "base32.js": { "version": "0.0.1", @@ -2650,7 +2649,6 @@ "version": "0.8.6", "resolved": "https://registry.npmjs.org/dtrace-provider/-/dtrace-provider-0.8.6.tgz", "integrity": "sha1-QooiOv4DQl0s1tY0f99AxmkDVj0=", - "optional": true, "requires": { "nan": "https://registry.npmjs.org/nan/-/nan-2.8.0.tgz" } @@ -6813,29 +6811,16 @@ } }, "readdirp": { - "version": "https://registry.npmjs.org/readdirp/-/readdirp-2.1.0.tgz", - "integrity": "sha512-LgQ8mdp6hbxJUZz27qxVl7gmFM/0DfHRO52c5RUbKAgMvr81tour7YYWW1JYNmrXyD/o0Myy9/DC3fUYkqnyzg==", - "dev": true, + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-2.1.0.tgz", + "integrity": "sha1-TtCtBg3zBzMAxIRANz9y0cxkLXg=", "requires": { - "graceful-fs": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.11.tgz", - "minimatch": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", - "readable-stream": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.3.tgz", - "set-immediate-shim": "https://registry.npmjs.org/set-immediate-shim/-/set-immediate-shim-1.0.1.tgz" - } - }, - "recursive-readdir": { - "version": "2.2.2", - "resolved": "https://registry.npmjs.org/recursive-readdir/-/recursive-readdir-2.2.2.tgz", - "integrity": "sha512-nRCcW9Sj7NuZwa2XvH9co8NPeXUBhZP7CRKJtU+cS6PW9FpCIFoI5ib0NT1ZrbNuPoRy0ylyCaUL8Gih4LSyFg==", - "requires": { - "minimatch": "3.0.4" + "graceful-fs": "4.1.11", + "minimatch": "3.0.4", + "readable-stream": "2.3.6", + "set-immediate-shim": "1.0.1" }, "dependencies": { - "balanced-match": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", - "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" - }, "brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -6850,6 +6835,26 @@ "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" }, + "core-util-is": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", + "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" + }, + "graceful-fs": { + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.11.tgz", + "integrity": "sha1-Dovf5NHduIVNZOBOp8AOKgJuVlg=" + }, + "inherits": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", + "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" + }, + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, "minimatch": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", @@ -6857,6 +6862,79 @@ "requires": { "brace-expansion": "1.1.11" } + }, + "process-nextick-args": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz", + "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==" + }, + "readable-stream": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", + "requires": { + "core-util-is": "1.0.2", + "inherits": "2.0.3", + "isarray": "1.0.0", + "process-nextick-args": "2.0.0", + "safe-buffer": "5.1.2", + "string_decoder": "1.1.1", + "util-deprecate": "1.0.2" + } + }, + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "requires": { + "safe-buffer": "5.1.2" + } + }, + "util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" + } + } + }, + "recursive-readdir": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/recursive-readdir/-/recursive-readdir-2.2.2.tgz", + "integrity": "sha512-nRCcW9Sj7NuZwa2XvH9co8NPeXUBhZP7CRKJtU+cS6PW9FpCIFoI5ib0NT1ZrbNuPoRy0ylyCaUL8Gih4LSyFg==", + "dev": true, + "requires": { + "minimatch": "3.0.4" + }, + "dependencies": { + "brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "requires": { + "balanced-match": "1.0.0", + "concat-map": "0.0.1" + } + }, + "concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", + "dev": true + }, + "minimatch": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", + "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "dev": true, + "requires": { + "brace-expansion": "1.1.11" + } } } }, @@ -7232,7 +7310,7 @@ "rimraf": { "version": "2.6.2", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.2.tgz", - "integrity": "sha512-lreewLK/BlghmxtfH36YYVg1i8IAce4TI7oao75I1g245+6BctqTVQiBP3YUJ9C6DQOXJmkYR9X9fCLtCOJc5w==", + "integrity": "sha1-LtgVDSShbqhlHm1u8PR8QVjOejY=", "requires": { "glob": "https://registry.npmjs.org/glob/-/glob-7.1.2.tgz" } @@ -7249,8 +7327,7 @@ "safe-json-stringify": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/safe-json-stringify/-/safe-json-stringify-1.1.0.tgz", - "integrity": "sha512-EzBtUaFH9bHYPc69wqjp0efJI/DPNHdFbGE3uIMn4sVbO0zx8vZ8cG4WKxQfOpUOKsQyGBiT2mTqnCw+6nLswA==", - "optional": true + "integrity": "sha512-EzBtUaFH9bHYPc69wqjp0efJI/DPNHdFbGE3uIMn4sVbO0zx8vZ8cG4WKxQfOpUOKsQyGBiT2mTqnCw+6nLswA==" }, "safer-buffer": { "version": "2.1.2", @@ -7641,9 +7718,9 @@ "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=" }, "set-immediate-shim": { - "version": "https://registry.npmjs.org/set-immediate-shim/-/set-immediate-shim-1.0.1.tgz", - "integrity": "sha1-SysbJ+uAip+NzEgaWOXlb1mfP2E=", - "dev": true + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/set-immediate-shim/-/set-immediate-shim-1.0.1.tgz", + "integrity": "sha1-SysbJ+uAip+NzEgaWOXlb1mfP2E=" }, "shebang-command": { "version": "https://registry.npmjs.org/shebang-command/-/shebang-command-1.2.0.tgz", diff --git a/package.json b/package.json index de6ffca5c..6b82ff0b6 100644 --- a/package.json +++ b/package.json @@ -57,7 +57,7 @@ "progress-stream": "^2.0.0", "proxy-middleware": "^0.15.0", "qrcode": "^1.2.0", - "recursive-readdir": "^2.2.2", + "readdirp": "^2.1.0", "request": "^2.87.0", "rimraf": "^2.6.2", "s3-block-read-stream": "^0.5.0", @@ -86,7 +86,7 @@ "mock-aws-s3": "git+https://github.com/cloudron-io/mock-aws-s3.git", "nock": "^9.0.14", "node-sass": "^4.6.1", - "readdirp": "https://registry.npmjs.org/readdirp/-/readdirp-2.1.0.tgz" + "recursive-readdir": "^2.2.2" }, "scripts": { "migrate_local": "DATABASE_URL=mysql://root:@localhost/box node_modules/.bin/db-migrate up", diff --git a/src/storage/filesystem.js b/src/storage/filesystem.js index 4daa8c73a..ea7d8a98e 100644 --- a/src/storage/filesystem.js +++ b/src/storage/filesystem.js @@ -14,12 +14,14 @@ exports = module.exports = { }; var assert = require('assert'), + async = require('async'), BackupsError = require('../backups.js').BackupsError, debug = require('debug')('box:storage/filesystem'), EventEmitter = require('events'), fs = require('fs'), mkdirp = require('mkdirp'), path = require('path'), + readdirp = require('readdirp'), safe = require('safetydance'), shell = require('../shell.js'); @@ -74,6 +76,34 @@ function download(apiConfig, sourceFilePath, callback) { callback(null, fileStream); } +function listDir(apiConfig, dir, batchSize, iteratorCallback, callback) { + assert.strictEqual(typeof apiConfig, 'object'); + assert.strictEqual(typeof dir, 'string'); + assert.strictEqual(typeof batchSize, 'number'); + assert.strictEqual(typeof iteratorCallback, 'function'); + assert.strictEqual(typeof callback, 'function'); + + var entries = []; + var entryStream = readdirp({ root: dir, entryType: 'files' }); + entryStream.on('data', function (data) { + entries.push(data.path); + if (entries.length < batchSize) return; + entryStream.pause(); + iteratorCallback(entries, function (error) { + if (error) return callback(error); + + entries = []; + entryStream.resume(); + }); + }); + entryStream.on('warn', function (error) { + debug('listDir: warning ', error); + }); + entryStream.on('end', function () { + iteratorCallback(entries, callback); + }); +} + function downloadDir(apiConfig, backupFilePath, destDir) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof backupFilePath, 'string'); @@ -83,7 +113,28 @@ function downloadDir(apiConfig, backupFilePath, destDir) { events.emit('progress', `downloadDir: ${backupFilePath} to ${destDir}`); - shell.exec('downloadDir', '/bin/cp', [ '-r', backupFilePath + '/.', destDir ], { }, function (error) { + function downloadFile(filePath, callback) { + const sourceFilePath = path.join(backupFilePath, filePath); + const destFilePath = path.join(destDir, filePath); + + mkdirp(path.dirname(destFilePath), function (error) { + if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message)); + + let sourceStream = fs.createReadStream(sourceFilePath); + sourceStream.on('error', callback); + + let destStream = fs.createWriteStream(destFilePath); + destStream.on('error', callback); + + events.emit('progress', `downloadDir: Copying ${sourceFilePath} to ${destFilePath}`); + + sourceStream.pipe(destStream, { end: true }).on('finish', callback); + }); + } + + listDir(apiConfig, backupFilePath, 1000, function (filePaths, done) { + async.each(filePaths, downloadFile, done); + }, function (error) { if (error) return events.emit('done', new BackupsError(BackupsError.EXTERNAL_ERROR, error.message)); events.emit('done', null);