implement remove dir in storage backends

src/storage/s3.js

@@ -5,7 +5,7 @@ exports = module.exports = {
     download: download,
     copy: copy,
 
-    removeMany: removeMany,
+    remove: remove,
 
     backupDone: backupDone,
 
@@ -17,6 +17,7 @@ exports = module.exports = {
 };
 
 var assert = require('assert'),
+    async = require('async'),
     AWS = require('aws-sdk'),
     BackupsError = require('../backups.js').BackupsError,
     debug = require('debug')('box:storage/s3'),
@@ -150,30 +151,49 @@ function copy(apiConfig, oldFilePath, newFilePath, callback) {
     });
 }
 
-function removeMany(apiConfig, filePaths, callback) {
+function remove(apiConfig, pathPrefix, callback) {
     assert.strictEqual(typeof apiConfig, 'object');
-    assert(Array.isArray(filePaths));
+    assert.strictEqual(typeof pathPrefix, 'string');
     assert.strictEqual(typeof callback, 'function');
 
     getBackupCredentials(apiConfig, function (error, credentials) {
         if (error) return callback(error);
 
-        var params = {
+        var s3 = new AWS.S3(credentials);
+        var listParams = {
             Bucket: apiConfig.bucket,
-            Delete: {
-                Objects: [ ] // { Key }
-            }
+            Prefix: pathPrefix
         };
 
-        filePaths.forEach(function (filePath) {
-            params.Delete.Objects.push({ Key: filePath });
-        });
+        async.forever(function listAndDelete(iteratorCallback) {
+            s3.listObjectsV2(listParams, function (error, listData) {
+                if (error) {
+                    debug('remove: Failed to list %s. Not fatal.', error);
+                    return iteratorCallback(error);
+                }
 
-        var s3 = new AWS.S3(credentials);
-        s3.deleteObjects(params, function (error, data) {
-            if (error) debug('removeMany: Unable to remove %s. Not fatal.', params.Key, error);
-            else debug('removeMany: Deleted: %j Errors: %j', data.Deleted, data.Errors);
+                var deleteParams = {
+                    Bucket: apiConfig.bucket,
+                    Delete: {
+                        Objects: listData.Contents.map(function (c) { return { Key: c.Key }; })
+                    }
+                };
 
+                s3.deleteObjects(deleteParams, function (error, deleteData) {
+                    if (error) {
+                        debug('remove: Unable to remove %s. Not fatal.', deleteParams.Key, error);
+                        return iteratorCallback(error);
+                    }
+                    debug(': Deleted: %j Errors: %j', deleteData.Deleted, deleteData.Errors);
+
+                    listParams.Marker = listData.Contents[listData.Contents.length - 1].Key; // NextMarker is returned only with delimiter
+
+                    if (deleteData.IsTruncated) return iteratorCallback();
+
+                    iteratorCallback(new Error('Done'));
+                });
+            });
+        }, function (/*ignoredError*/) {
             callback(null);
         });
     });