storage: standardize the listDir API

src/storage/filesystem.js

@@ -7,6 +7,8 @@ exports = module.exports = {
 
     copy: copy,
 
+    listDir: listDir,
+
     remove: remove,
     removeDir: removeDir,
 
@@ -86,7 +88,7 @@ function listDir(apiConfig, dir, batchSize, iteratorCallback, callback) {
     var entries = [];
     var entryStream = readdirp({ root: dir, entryType: 'files' });
     entryStream.on('data', function (data) {
-        entries.push(data.path);
+        entries.push({ path: data.path, size: data.stat.size });
         if (entries.length < batchSize) return;
         entryStream.pause();
         iteratorCallback(entries, function (error) {
@@ -113,9 +115,9 @@ function downloadDir(apiConfig, backupFilePath, destDir) {
 
     events.emit('progress', `downloadDir: ${backupFilePath} to ${destDir}`);
 
-    function downloadFile(filePath, callback) {
-        const sourceFilePath = path.join(backupFilePath, filePath);
-        const destFilePath = path.join(destDir, filePath);
+    function downloadFile(file, callback) {
+        const sourceFilePath = path.join(backupFilePath, file.path);
+        const destFilePath = path.join(destDir, file.path);
 
         mkdirp(path.dirname(destFilePath), function (error) {
             if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message));
@@ -132,8 +134,8 @@ function downloadDir(apiConfig, backupFilePath, destDir) {
         });
     }
 
-    listDir(apiConfig, backupFilePath, 1000, function (filePaths, done) {
-        async.each(filePaths, downloadFile, done);
+    listDir(apiConfig, backupFilePath, 1000, function (entries, done) {
+        async.each(entries, downloadFile, done);
     }, function (error) {
         if (error) return events.emit('done', new BackupsError(BackupsError.EXTERNAL_ERROR, error.message));
 

src/storage/interface.js

@@ -17,6 +17,8 @@ exports = module.exports = {
     downloadDir: downloadDir,
     copy: copy,
 
+    listDir: listDir,
+
     remove: remove,
     removeDir: removeDir,
 
@@ -67,6 +69,16 @@ function copy(apiConfig, oldFilePath, newFilePath) {
     return events;
 }
 
+function listDir(apiConfig, dir, batchSize, iteratorCallback, callback) {
+    assert.strictEqual(typeof apiConfig, 'object');
+    assert.strictEqual(typeof dir, 'string');
+    assert.strictEqual(typeof batchSize, 'number');
+    assert.strictEqual(typeof iteratorCallback, 'function');
+    assert.strictEqual(typeof callback, 'function');
+
+    callback(new Error('not implemented'));
+}
+
 function remove(apiConfig, filename, callback) {
     assert.strictEqual(typeof apiConfig, 'object');
     assert.strictEqual(typeof filename, 'string');

src/storage/noop.js

@@ -6,6 +6,8 @@ exports = module.exports = {
     downloadDir: downloadDir,
     copy: copy,
 
+    listDir: listDir,
+
     remove: remove,
     removeDir: removeDir,
 
@@ -37,6 +39,16 @@ function download(apiConfig, backupFilePath, callback) {
     callback(new Error('Cannot download from noop backend'));
 }
 
+function listDir(apiConfig, dir, batchSize, iteratorCallback, callback) {
+    assert.strictEqual(typeof apiConfig, 'object');
+    assert.strictEqual(typeof dir, 'string');
+    assert.strictEqual(typeof batchSize, 'number');
+    assert.strictEqual(typeof iteratorCallback, 'function');
+    assert.strictEqual(typeof callback, 'function');
+
+    callback();
+}
+
 function downloadDir(apiConfig, backupFilePath, destDir) {
     assert.strictEqual(typeof apiConfig, 'object');
     assert.strictEqual(typeof backupFilePath, 'string');

src/storage/s3.js

@@ -197,10 +197,7 @@ function listDir(apiConfig, dir, iteratorCallback, callback) {
 
         async.forever(function listAndDownload(foreverCallback) {
             s3.listObjects(listParams, function (error, listData) {
-                if (error) {
-                    debug('remove: Failed to list %s. Not fatal.', error);
-                    return foreverCallback(error);
-                }
+                if (error) return foreverCallback(error);
 
                 if (listData.Contents.length === 0) return foreverCallback(new Error('Done'));