jjkiers/cloudron-box
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

implement remove dir in storage backends

Browse Source
This commit is contained in:
Girish Ramakrishnan
2017-09-23 11:09:36 -07:00
parent e39a5c8872
commit e43413e063
7 changed files with 107 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/storage/caas.js
+37 -14
View File
@@ -4,7 +4,7 @@ exports = module.exports = {
upload: upload,
download: download,
copy: copy,
removeMany: removeMany,
remove: remove,
backupDone: backupDone,
@@ -12,6 +12,7 @@ exports = module.exports = {
};
var assert = require('assert'),
async = require('async'),
AWS = require('aws-sdk'),
BackupsError = require('../backups.js').BackupsError,
config = require('../config.js'),
@@ -141,29 +142,51 @@ function copy(apiConfig, oldFilePath, newFilePath, callback) {
});
}
function removeMany(apiConfig, filePaths, callback) {
function remove(apiConfig, pathPrefix, callback) {
assert.strictEqual(typeof apiConfig, 'object');
assert(Array.isArray(filePaths));
assert.strictEqual(typeof pathPrefix, 'string');
assert.strictEqual(typeof callback, 'function');
getBackupCredentials(apiConfig, function (error, credentials) {
if (error) return callback(error);
var params = {
var s3 = new AWS.S3(credentials);
var listParams = {
Bucket: apiConfig.bucket,
Delete: {
Objects: [ ] // { Key }
}
Prefix: pathPrefix
};
filePaths.forEach(function (filePath) {
params.Delete.Objects.push({ Key: filePath });
});
async.forever(function listAndDelete(iteratorCallback) {
s3.listObjectsV2(listParams, function (error, listData) {
if (error) {
debug('remove: Failed to list %s. Not fatal.', error);
return iteratorCallback(error);
}
var s3 = new AWS.S3(credentials);
s3.deleteObjects(params, function (error, data) {
if (error) debug('Unable to remove %s. Not fatal.', params.Key, error);
else debug('removeMany: Deleted: %j Errors: %j', data.Deleted, data.Errors);
var deleteParams = {
Bucket: apiConfig.bucket,
Delete: {
Objects: listData.Contents.map(function (c) { return { Key: c.Key }; })
}
};
s3.deleteObjects(deleteParams, function (error, deleteData) {
if (error) {
debug('remove: Unable to remove %s. Not fatal.', deleteParams.Key, error);
return iteratorCallback(error);
}
debug('remove: Deleted: %j Errors: %j', deleteData.Deleted, deleteData.Errors);
listParams.Marker = listData[listData.length - 1].Key; // NextMarker is returned only with delimiter
if (deleteData.IsTruncated) return iteratorCallback();
iteratorCallback(new Error('Done'));
});
});
}, function (/*ignoredError*/) {
if (error.message === 'Done') return callback(null);
callback(null);
});
src/storage/filesystem.js
+8 -11
View File
@@ -5,7 +5,7 @@ exports = module.exports = {
download: download,
copy: copy,
removeMany: removeMany,
remove: remove,
backupDone: backupDone,
@@ -13,7 +13,6 @@ exports = module.exports = {
};
var assert = require('assert'),
async = require('async'),
BackupsError = require('../backups.js').BackupsError,
config = require('../config.js'),
debug = require('debug')('box:storage/filesystem'),
@@ -95,20 +94,18 @@ function copy(apiConfig, oldFilePath, newFilePath, callback) {
});
}
function removeMany(apiConfig, filePaths, callback) {
function remove(apiConfig, pathPrefix, callback) {
assert.strictEqual(typeof apiConfig, 'object');
assert(Array.isArray(filePaths));
assert.strictEqual(typeof pathPrefix, 'string');
assert.strictEqual(typeof callback, 'function');
async.eachSeries(filePaths, function (filePath, iteratorCallback) {
if (!safe.fs.unlinkSync(filePath)) {
debug('removeMany: Unable to remove %s : %s', filePath, safe.error.message);
}
shell.exec('remove', '/bin/rm', [ '-rf', pathPrefix ], { }, function (error) {
if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message));
safe.fs.rmdirSync(path.dirname(filePath)); // try to cleanup empty directories
safe.fs.rmdirSync(path.dirname(pathPrefix)); // try to cleanup empty directories
iteratorCallback();
}, callback);
callback();
});
}
function testConfig(apiConfig, callback) {
src/storage/interface.js
+3 -3
View File
@@ -11,7 +11,7 @@ exports = module.exports = {
download: download,
copy: copy,
removeMany: removeMany,
remove: remove,
backupDone: backupDone,
@@ -51,9 +51,9 @@ function copy(apiConfig, oldFilePath, newFilePath, callback) {
callback(new Error('not implemented'));
}
function removeMany(apiConfig, filePaths, callback) {
function remove(apiConfig, pathPrefix, callback) {
assert.strictEqual(typeof apiConfig, 'object');
assert(Array.isArray(filePaths));
assert.strictEqual(typeof pathPrefix, 'string');
assert.strictEqual(typeof callback, 'function');
// Result: none
src/storage/noop.js
+4 -4
View File
@@ -5,7 +5,7 @@ exports = module.exports = {
download: download,
copy: copy,
removeMany: removeMany,
remove: remove,
backupDone: backupDone,
@@ -47,12 +47,12 @@ function copy(apiConfig, oldFilePath, newFilePath, callback) {
callback();
}
function removeMany(apiConfig, filePaths, callback) {
function remove(apiConfig, pathPrefix, callback) {
assert.strictEqual(typeof apiConfig, 'object');
assert(Array.isArray(filePaths));
assert.strictEqual(typeof pathPrefix, 'string');
assert.strictEqual(typeof callback, 'function');
debug('removeMany: %j', filePaths);
debug('remove: %s', pathPrefix);
callback();
}
src/storage/s3.js
+34 -14
View File
@@ -5,7 +5,7 @@ exports = module.exports = {
download: download,
copy: copy,
removeMany: removeMany,
remove: remove,
backupDone: backupDone,
@@ -17,6 +17,7 @@ exports = module.exports = {
};
var assert = require('assert'),
async = require('async'),
AWS = require('aws-sdk'),
BackupsError = require('../backups.js').BackupsError,
debug = require('debug')('box:storage/s3'),
@@ -150,30 +151,49 @@ function copy(apiConfig, oldFilePath, newFilePath, callback) {
});
}
function removeMany(apiConfig, filePaths, callback) {
function remove(apiConfig, pathPrefix, callback) {
assert.strictEqual(typeof apiConfig, 'object');
assert(Array.isArray(filePaths));
assert.strictEqual(typeof pathPrefix, 'string');
assert.strictEqual(typeof callback, 'function');
getBackupCredentials(apiConfig, function (error, credentials) {
if (error) return callback(error);
var params = {
var s3 = new AWS.S3(credentials);
var listParams = {
Bucket: apiConfig.bucket,
Delete: {
Objects: [ ] // { Key }
}
Prefix: pathPrefix
};
filePaths.forEach(function (filePath) {
params.Delete.Objects.push({ Key: filePath });
});
async.forever(function listAndDelete(iteratorCallback) {
s3.listObjectsV2(listParams, function (error, listData) {
if (error) {
debug('remove: Failed to list %s. Not fatal.', error);
return iteratorCallback(error);
}
var s3 = new AWS.S3(credentials);
s3.deleteObjects(params, function (error, data) {
if (error) debug('removeMany: Unable to remove %s. Not fatal.', params.Key, error);
else debug('removeMany: Deleted: %j Errors: %j', data.Deleted, data.Errors);
var deleteParams = {
Bucket: apiConfig.bucket,
Delete: {
Objects: listData.Contents.map(function (c) { return { Key: c.Key }; })
}
};
s3.deleteObjects(deleteParams, function (error, deleteData) {
if (error) {
debug('remove: Unable to remove %s. Not fatal.', deleteParams.Key, error);
return iteratorCallback(error);
}
debug(': Deleted: %j Errors: %j', deleteData.Deleted, deleteData.Errors);
listParams.Marker = listData.Contents[listData.Contents.length - 1].Key; // NextMarker is returned only with delimiter
if (deleteData.IsTruncated) return iteratorCallback();
iteratorCallback(new Error('Done'));
});
});
}, function (/*ignoredError*/) {
callback(null);
});
});