diff --git a/src/backups.js b/src/backups.js index f7de3d479..6afbbbf69 100644 --- a/src/backups.js +++ b/src/backups.js @@ -208,7 +208,7 @@ function decryptFilePath(filePath, key) { part = part + Array(part.length % 4).join('='); // add back = padding part = part.replace(/-/g, '/'); // replace with '/' - var decrypt = crypto.createDecipher('aes-256-cbc', 'incremental'); + var decrypt = crypto.createDecipher('aes-256-cbc', key); let text = decrypt.update(Buffer.from(part, 'base64')); text = Buffer.concat([ text, decrypt.final() ]); return text.toString('utf8'); @@ -488,16 +488,16 @@ function downloadDir(backupConfig, backupFilePath, destDir, callback) { assert.strictEqual(typeof destDir, 'string'); assert.strictEqual(typeof callback, 'function'); - log(`downloadDir: ${backupFilePath} to ${destDir}`); + debug(`downloadDir: ${backupFilePath} to ${destDir}`); function downloadFile(entry, callback) { - const sourceFilePath = path.join(backupFilePath, entry.path); - const destFilePath = path.join(destDir, backupConfig.key ? decryptFilePath(entry.path, backupConfig.key) : entry.path); + const relativePath = path.relative(backupFilePath, entry.fullPath); + const destFilePath = path.join(destDir, backupConfig.key ? decryptFilePath(relativePath, backupConfig.key) : relativePath); mkdirp(path.dirname(destFilePath), function (error) { if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message)); - api(backupConfig.provider).download(backupConfig, sourceFilePath, function (error, sourceStream) { + api(backupConfig.provider).download(backupConfig, entry.fullPath, function (error, sourceStream) { if (error) return callback(error); sourceStream.on('error', callback); @@ -505,7 +505,7 @@ function downloadDir(backupConfig, backupFilePath, destDir, callback) { let destStream = createWriteStream(destFilePath, backupConfig.key || null); destStream.on('error', callback); - log(`downloadDir: Copying ${sourceFilePath} to ${destFilePath}`); + debug(`downloadDir: Copying ${entry.fullPath} to ${destFilePath}`); sourceStream.pipe(destStream, { end: true }).on('finish', callback); }); diff --git a/src/storage/filesystem.js b/src/storage/filesystem.js index 7cac6bbc6..f5b6edfc3 100644 --- a/src/storage/filesystem.js +++ b/src/storage/filesystem.js @@ -86,7 +86,7 @@ function listDir(apiConfig, dir, batchSize, iteratorCallback, callback) { var entries = []; var entryStream = readdirp({ root: dir, entryType: 'files' }); entryStream.on('data', function (data) { - entries.push({ path: data.path, size: data.stat.size }); + entries.push({ fullPath: data.fullPath }); if (entries.length < batchSize) return; entryStream.pause(); iteratorCallback(entries, function (error) { diff --git a/src/storage/gcs.js b/src/storage/gcs.js index c284acaeb..1983d32f1 100644 --- a/src/storage/gcs.js +++ b/src/storage/gcs.js @@ -3,9 +3,10 @@ exports = module.exports = { upload: upload, download: download, - downloadDir: downloadDir, copy: copy, + listDir: listDir, + remove: remove, removeDir: removeDir, @@ -21,9 +22,7 @@ var assert = require('assert'), BackupsError = require('../backups.js').BackupsError, debug = require('debug')('box:storage/gcs'), EventEmitter = require('events'), - fs = require('fs'), GCS = require('@google-cloud/storage'), - mkdirp = require('mkdirp'), PassThrough = require('stream').PassThrough, path = require('path'); @@ -115,24 +114,17 @@ function listDir(apiConfig, backupFilePath, batchSize, iteratorCallback, callbac async.forever(function listAndDownload(foreverCallback) { bucket.getFiles(query, function (error, files, nextQuery) { - if (error) { - debug('remove: Failed to list %s. Not fatal.', error); - return foreverCallback(error); - } + if (error) return foreverCallback(error); if (files.length === 0) return foreverCallback(new Error('Done')); - debug('emitting ' + files.length + ' files found: ' + files.map(function(f) { return f.name; }).join(',')); - iteratorCallback(files, function (error) { - if (error) { - debug(`listDir page handled unsuccessfully ${error}`); - return foreverCallback(error); - } - + const entries = files.map(function (f) { return { fullPath: f.name }; }); + iteratorCallback(entries, function (error) { + if (error) return foreverCallback(error); if (!nextQuery) return foreverCallback(new Error('Done')); query = nextQuery; - debug(`listDir next page token ${query.pageToken}`); + foreverCallback(); }); }); @@ -143,53 +135,6 @@ function listDir(apiConfig, backupFilePath, batchSize, iteratorCallback, callbac }); } -function downloadDir(apiConfig, backupFilePath, destDir) { - assert.strictEqual(typeof apiConfig, 'object'); - assert.strictEqual(typeof backupFilePath, 'string'); - assert.strictEqual(typeof destDir, 'string'); - - var events = new EventEmitter(); - var total = 0; - - function downloadFile(file, iteratorCallback) { - var relativePath = path.relative(backupFilePath, file.name); - - events.emit('progress', `Downloading ${relativePath}`); - - mkdirp(path.dirname(path.join(destDir, relativePath)), function (error) { - if (error) return iteratorCallback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message)); - - download(apiConfig, file.name, function (error, sourceStream) { - if (error) return iteratorCallback(error); - - var destStream = fs.createWriteStream(path.join(destDir, relativePath)); - - destStream.on('open', function () { - sourceStream.pipe(destStream); - }); - - destStream.on('error', function (error) { - iteratorCallback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message)); - }); - - destStream.on('finish', iteratorCallback); - }); - }); - } - - const concurrency = 10, batchSize = -1; - - listDir(apiConfig, backupFilePath, batchSize, function (files, done) { - total += files.length; - async.eachLimit(files, concurrency, downloadFile, done); - }, function (error) { - events.emit('progress', `Downloaded ${total} files`); - events.emit('done', error); - }); - - return events; -} - function copy(apiConfig, oldFilePath, newFilePath) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof oldFilePath, 'string'); @@ -197,16 +142,15 @@ function copy(apiConfig, oldFilePath, newFilePath) { var events = new EventEmitter(), retryCount = 0; - function copyFile(file, iteratorCallback) { + function copyFile(entry, iteratorCallback) { + var relativePath = path.relative(oldFilePath, entry.fullPath); - var relativePath = path.relative(oldFilePath, file.name); + getBucket(apiConfig).file(entry.fullPath).copy(path.join(newFilePath, relativePath), function(error) { + if (error) debug('copyBackup: gcs copy error', error); - file.copy(path.join(newFilePath, relativePath), function(error) { if (error && error.code === 404) return iteratorCallback(new BackupsError(BackupsError.NOT_FOUND, 'Old backup not found')); - if (error) { - debug('copyBackup: gcs copy error', error); - return iteratorCallback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message)); - } + if (error) return iteratorCallback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message)); + iteratorCallback(null); }); @@ -216,17 +160,16 @@ function copy(apiConfig, oldFilePath, newFilePath) { const batchSize = -1; var total = 0, concurrency = 4; - listDir(apiConfig, oldFilePath, batchSize, function (files, done) { - total += files.length; + listDir(apiConfig, oldFilePath, batchSize, function (entries, done) { + total += entries.length; if (retryCount === 0) concurrency = Math.min(concurrency + 1, 10); else concurrency = Math.max(concurrency - 1, 5); events.emit('progress', `${retryCount} errors. concurrency set to ${concurrency}`); retryCount = 0; - async.eachLimit(files, concurrency, copyFile, done); + async.eachLimit(entries, concurrency, copyFile, done); }, function (error) { events.emit('progress', `Copied ${total} files`); - events.emit('done', error); }); @@ -240,9 +183,9 @@ function remove(apiConfig, filename, callback) { getBucket(apiConfig) .file(filename) - .delete(function(e) { - if (e) debug('removeBackups: Unable to remove %s (%s). Not fatal.', filename, e.message); - else debug('removeBackups: Deleted: %s', filename); + .delete(function (error) { + if (error) debug('removeBackups: Unable to remove %s (%s). Not fatal.', filename, error.message); + callback(null); }); } @@ -256,14 +199,16 @@ function removeDir(apiConfig, pathPrefix) { const batchSize = 1; var total = 0, concurrency = 4; - listDir(apiConfig, pathPrefix, batchSize, function (files, done) { - total += files.length; + listDir(apiConfig, pathPrefix, batchSize, function (entries, done) { + total += entries.length; if (retryCount === 0) concurrency = Math.min(concurrency + 1, 10); else concurrency = Math.max(concurrency - 1, 5); events.emit('progress', `${retryCount} errors. concurrency set to ${concurrency}`); retryCount = 0; - async.eachLimit(files, concurrency, remove.bind(null, apiConfig), done); + async.eachLimit(entries, concurrency, function (entry, iteratorCallback) { + remove(apiConfig, entry.fullPath, iteratorCallback); + }, done); }, function (error) { events.emit('progress', `Deleted ${total} files`);