diff --git a/src/storage/filesystem.js b/src/storage/filesystem.js index ea7d8a98e..2bdf7ff31 100644 --- a/src/storage/filesystem.js +++ b/src/storage/filesystem.js @@ -7,6 +7,8 @@ exports = module.exports = { copy: copy, + listDir: listDir, + remove: remove, removeDir: removeDir, @@ -86,7 +88,7 @@ function listDir(apiConfig, dir, batchSize, iteratorCallback, callback) { var entries = []; var entryStream = readdirp({ root: dir, entryType: 'files' }); entryStream.on('data', function (data) { - entries.push(data.path); + entries.push({ path: data.path, size: data.stat.size }); if (entries.length < batchSize) return; entryStream.pause(); iteratorCallback(entries, function (error) { @@ -113,9 +115,9 @@ function downloadDir(apiConfig, backupFilePath, destDir) { events.emit('progress', `downloadDir: ${backupFilePath} to ${destDir}`); - function downloadFile(filePath, callback) { - const sourceFilePath = path.join(backupFilePath, filePath); - const destFilePath = path.join(destDir, filePath); + function downloadFile(file, callback) { + const sourceFilePath = path.join(backupFilePath, file.path); + const destFilePath = path.join(destDir, file.path); mkdirp(path.dirname(destFilePath), function (error) { if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message)); @@ -132,8 +134,8 @@ function downloadDir(apiConfig, backupFilePath, destDir) { }); } - listDir(apiConfig, backupFilePath, 1000, function (filePaths, done) { - async.each(filePaths, downloadFile, done); + listDir(apiConfig, backupFilePath, 1000, function (entries, done) { + async.each(entries, downloadFile, done); }, function (error) { if (error) return events.emit('done', new BackupsError(BackupsError.EXTERNAL_ERROR, error.message)); diff --git a/src/storage/interface.js b/src/storage/interface.js index 378385503..be24cba6f 100644 --- a/src/storage/interface.js +++ b/src/storage/interface.js @@ -17,6 +17,8 @@ exports = module.exports = { downloadDir: downloadDir, copy: copy, + listDir: listDir, + remove: remove, removeDir: removeDir, @@ -67,6 +69,16 @@ function copy(apiConfig, oldFilePath, newFilePath) { return events; } +function listDir(apiConfig, dir, batchSize, iteratorCallback, callback) { + assert.strictEqual(typeof apiConfig, 'object'); + assert.strictEqual(typeof dir, 'string'); + assert.strictEqual(typeof batchSize, 'number'); + assert.strictEqual(typeof iteratorCallback, 'function'); + assert.strictEqual(typeof callback, 'function'); + + callback(new Error('not implemented')); +} + function remove(apiConfig, filename, callback) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof filename, 'string'); diff --git a/src/storage/noop.js b/src/storage/noop.js index 39d1c2059..d6f7ecbbb 100644 --- a/src/storage/noop.js +++ b/src/storage/noop.js @@ -6,6 +6,8 @@ exports = module.exports = { downloadDir: downloadDir, copy: copy, + listDir: listDir, + remove: remove, removeDir: removeDir, @@ -37,6 +39,16 @@ function download(apiConfig, backupFilePath, callback) { callback(new Error('Cannot download from noop backend')); } +function listDir(apiConfig, dir, batchSize, iteratorCallback, callback) { + assert.strictEqual(typeof apiConfig, 'object'); + assert.strictEqual(typeof dir, 'string'); + assert.strictEqual(typeof batchSize, 'number'); + assert.strictEqual(typeof iteratorCallback, 'function'); + assert.strictEqual(typeof callback, 'function'); + + callback(); +} + function downloadDir(apiConfig, backupFilePath, destDir) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof backupFilePath, 'string'); diff --git a/src/storage/s3.js b/src/storage/s3.js index eec4b059e..d3c757e2d 100644 --- a/src/storage/s3.js +++ b/src/storage/s3.js @@ -197,10 +197,7 @@ function listDir(apiConfig, dir, iteratorCallback, callback) { async.forever(function listAndDownload(foreverCallback) { s3.listObjects(listParams, function (error, listData) { - if (error) { - debug('remove: Failed to list %s. Not fatal.', error); - return foreverCallback(error); - } + if (error) return foreverCallback(error); if (listData.Contents.length === 0) return foreverCallback(new Error('Done'));