diff --git a/src/backupcleaner.js b/src/backupcleaner.js index b46764b7b..95bc8a663 100644 --- a/src/backupcleaner.js +++ b/src/backupcleaner.js @@ -87,34 +87,27 @@ async function removeBackup(backupConfig, backup, progressCallback) { const backupFilePath = storage.getBackupFilePath(backupConfig, backup.remotePath, backup.format); - return new Promise((resolve) => { - function done(error) { - if (error) { - debug('removeBackup: error removing backup %j : %s', backup, error.message); - return resolve(); - } + let removeError; + if (backup.format ==='tgz') { + progressCallback({ message: `${backup.remotePath}: Removing ${backupFilePath}`}); + [removeError] = await safe(storage.api(backupConfig.provider).remove(backupConfig, backupFilePath)); + } else { + progressCallback({ message: `${backup.remotePath}: Removing directory ${backupFilePath}`}); + [removeError] = await safe(storage.api(backupConfig.provider).removeDir(backupConfig, backupFilePath, progressCallback)); + } - // prune empty directory if possible - storage.api(backupConfig.provider).remove(backupConfig, path.dirname(backupFilePath), async function (error) { - if (error) debug('removeBackup: unable to prune backup directory %s : %s', path.dirname(backupFilePath), error.message); + if (removeError) { + debug('removeBackup: error removing backup %j : %s', backup, removeError.message); + return; + } - const [delError] = await safe(backups.del(backup.id)); - if (delError) debug(`removeBackup: error removing ${backup.id} from database`, delError); - else debug(`removeBackup: removed ${backup.remotePath}`); + // prune empty directory if possible + const [pruneError] = await safe(storage.api(backupConfig.provider).remove(backupConfig, path.dirname(backupFilePath))); + if (pruneError) debug('removeBackup: unable to prune backup directory %s : %s', path.dirname(backupFilePath), pruneError.message); - resolve(); - }); - } - - if (backup.format ==='tgz') { - progressCallback({ message: `${backup.id}: Removing ${backupFilePath}`}); - storage.api(backupConfig.provider).remove(backupConfig, backupFilePath, done); - } else { - const events = storage.api(backupConfig.provider).removeDir(backupConfig, backupFilePath); - events.on('progress', (message) => progressCallback({ message: `${backup.remotePath}: ${message}` })); - events.on('done', done); - } - }); + const [delError] = await safe(backups.del(backup.id)); + if (delError) debug(`removeBackup: error removing ${backup.id} from database`, delError); + else debug(`removeBackup: removed ${backup.remotePath}`); } async function cleanupAppBackups(backupConfig, referencedBackupIds, progressCallback) { @@ -251,29 +244,23 @@ async function cleanupSnapshots(backupConfig) { delete info.box; + const progressCallback = (progress) => { debug(`cleanupSnapshots: ${progress.message}`); }; + for (const appId of Object.keys(info)) { const app = await apps.get(appId); if (app) continue; // app is still installed - await new Promise((resolve) => { - async function done(/* ignoredError */) { - safe.fs.unlinkSync(path.join(paths.BACKUP_INFO_DIR, `${appId}.sync.cache`)); - safe.fs.unlinkSync(path.join(paths.BACKUP_INFO_DIR, `${appId}.sync.cache.new`)); + if (info[appId].format ==='tgz') { + await safe(storage.api(backupConfig.provider).remove(backupConfig, storage.getBackupFilePath(backupConfig, `snapshot/app_${appId}`, info[appId].format)), { debug }); + } else { + await safe(storage.api(backupConfig.provider).removeDir(backupConfig, storage.getBackupFilePath(backupConfig, `snapshot/app_${appId}`, info[appId].format), progressCallback), { debug }); + } - await safe(backups.setSnapshotInfo(appId, null /* info */), { debug }); - debug(`cleanupSnapshots: cleaned up snapshot of app ${appId}`); + safe.fs.unlinkSync(path.join(paths.BACKUP_INFO_DIR, `${appId}.sync.cache`)); + safe.fs.unlinkSync(path.join(paths.BACKUP_INFO_DIR, `${appId}.sync.cache.new`)); - resolve(); - } - - if (info[appId].format ==='tgz') { - storage.api(backupConfig.provider).remove(backupConfig, storage.getBackupFilePath(backupConfig, `snapshot/app_${appId}`, info[appId].format), done); - } else { - const events = storage.api(backupConfig.provider).removeDir(backupConfig, storage.getBackupFilePath(backupConfig, `snapshot/app_${appId}`, info[appId].format)); - events.on('progress', function (detail) { debug(`cleanupSnapshots: ${detail}`); }); - events.on('done', done); - } - }); + await safe(backups.setSnapshotInfo(appId, null /* info */), { debug }); + debug(`cleanupSnapshots: cleaned up snapshot of app ${appId}`); } debug('cleanupSnapshots: done'); diff --git a/src/backuptask.js b/src/backuptask.js index 791b86abb..998eefa9f 100644 --- a/src/backuptask.js +++ b/src/backuptask.js @@ -325,7 +325,7 @@ function sync(backupConfig, remotePath, dataLayout, progressCallback, callback) // the number here has to take into account the s3.upload partSize (which is 10MB). So 20=200MB const concurrency = backupConfig.syncConcurrency || (backupConfig.provider === 's3' ? 20 : 10); - syncer.sync(dataLayout, function processTask(task, iteratorCallback) { + syncer.sync(dataLayout, async function processTask(task, iteratorCallback) { debug('sync: processing task: %j', task); // the empty task.path is special to signify the directory const destPath = task.path && backupConfig.encryption ? encryptFilePath(task.path, backupConfig.encryption) : task.path; @@ -333,12 +333,12 @@ function sync(backupConfig, remotePath, dataLayout, progressCallback, callback) if (task.operation === 'removedir') { debug(`Removing directory ${backupFilePath}`); - return storage.api(backupConfig.provider).removeDir(backupConfig, backupFilePath) - .on('progress', (message) => progressCallback({ message })) - .on('done', iteratorCallback); + const [error] = await safe(storage.api(backupConfig.provider).removeDir(backupConfig, backupFilePath, progressCallback)); + return iteratorCallback(error); } else if (task.operation === 'remove') { debug(`Removing ${backupFilePath}`); - return storage.api(backupConfig.provider).remove(backupConfig, backupFilePath, iteratorCallback); + const [error] = await safe(storage.api(backupConfig.provider).remove(backupConfig, backupFilePath)); + return iteratorCallback(error); } let retryCount = 0; diff --git a/src/storage/filesystem.js b/src/storage/filesystem.js index ccb114cdc..2244efe9f 100644 --- a/src/storage/filesystem.js +++ b/src/storage/filesystem.js @@ -227,38 +227,29 @@ function copy(apiConfig, oldFilePath, newFilePath) { return events; } -function remove(apiConfig, filename, callback) { +async function remove(apiConfig, filename) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof filename, 'string'); - assert.strictEqual(typeof callback, 'function'); - var stat = safe.fs.statSync(filename); - if (!stat) return callback(); + const stat = safe.fs.statSync(filename); + if (!stat) return; if (stat.isFile()) { - if (!safe.fs.unlinkSync(filename)) return callback(new BoxError(BoxError.EXTERNAL_ERROR, safe.error.message)); + if (!safe.fs.unlinkSync(filename)) throw new BoxError(BoxError.EXTERNAL_ERROR, safe.error.message); } else if (stat.isDirectory()) { - if (!safe.fs.rmSync(filename, { recursive: true })) return callback(new BoxError(BoxError.EXTERNAL_ERROR, safe.error.message)); + if (!safe.fs.rmSync(filename, { recursive: true })) throw new BoxError(BoxError.EXTERNAL_ERROR, safe.error.message); } - - callback(null); } -function removeDir(apiConfig, pathPrefix) { +async function removeDir(apiConfig, pathPrefix, progressCallback) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof pathPrefix, 'string'); + assert.strictEqual(typeof progressCallback, 'function'); - var events = new EventEmitter(); + progressCallback({ message: `Removing directory ${pathPrefix}` }); - process.nextTick(() => events.emit('progress', `Removing directory ${pathPrefix}`)); - - shell.spawn('removeDir', '/bin/rm', [ '-rf', pathPrefix ], { }, function (error) { - if (error) return events.emit('done', new BoxError(BoxError.EXTERNAL_ERROR, error.message)); - - events.emit('done', null); - }); - - return events; + const [error] = await safe(shell.promises.spawn('removeDir', '/bin/rm', [ '-rf', pathPrefix ], { })); + if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, error.message); } function validateBackupTarget(folder) { diff --git a/src/storage/gcs.js b/src/storage/gcs.js index d8247a704..506cb6732 100644 --- a/src/storage/gcs.js +++ b/src/storage/gcs.js @@ -34,7 +34,8 @@ const assert = require('assert'), EventEmitter = require('events'), PassThrough = require('stream').PassThrough, path = require('path'), - safe = require('safetydance'); + safe = require('safetydance'), + util = require('util'); let GCS = require('@google-cloud/storage').Storage; @@ -221,44 +222,32 @@ function copy(apiConfig, oldFilePath, newFilePath) { return events; } -function remove(apiConfig, filename, callback) { +async function remove(apiConfig, filename) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof filename, 'string'); - assert.strictEqual(typeof callback, 'function'); - getBucket(apiConfig) - .file(filename) - .delete(function (error) { - if (error) debug('removeBackups: Unable to remove %s (%s). Not fatal.', filename, error.message); - - callback(null); - }); + const [error] = await safe(getBucket(apiConfig).file(filename).delete()); + if (error) debug('removeBackups: Unable to remove %s (%s). Not fatal.', filename, error.message); } -function removeDir(apiConfig, pathPrefix) { +async function removeDir(apiConfig, pathPrefix, progressCallback) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof pathPrefix, 'string'); - - var events = new EventEmitter(); + assert.strictEqual(typeof progressCallback, 'function'); const batchSize = 1000, concurrency = apiConfig.deleteConcurrency || 10; // https://googleapis.dev/nodejs/storage/latest/Bucket.html#deleteFiles - var total = 0; + let total = 0; - listDir(apiConfig, pathPrefix, batchSize, function (entries, done) { + const listDirAsync = util.promisify(listDir); + + await listDirAsync(apiConfig, pathPrefix, batchSize, function (entries, done) { total += entries.length; - events.emit('progress', `Removing ${entries.length} files from ${entries[0].fullPath} to ${entries[entries.length-1].fullPath}. total: ${total}`); + progressCallback({ message: `Removing ${entries.length} files from ${entries[0].fullPath} to ${entries[entries.length-1].fullPath}. total: ${total}` }); - async.eachLimit(entries, concurrency, function (entry, iteratorCallback) { - remove(apiConfig, entry.fullPath, iteratorCallback); - }, done); - }, function (error) { - events.emit('progress', `Deleted ${total} files`); - - process.nextTick(() => events.emit('done', error)); + async.eachLimit(entries, concurrency, async (entry) => await remove(apiConfig, entry.fullPath), done); }); - - return events; + progressCallback({ progress: `Deleted ${total} files` }); } async function remount(apiConfig) { diff --git a/src/storage/interface.js b/src/storage/interface.js index 19d959b56..b59d998a5 100644 --- a/src/storage/interface.js +++ b/src/storage/interface.js @@ -119,24 +119,21 @@ function listDir(apiConfig, dir, batchSize, iteratorCallback, callback) { callback(new BoxError(BoxError.NOT_IMPLEMENTED, 'listDir is not implemented')); } -function remove(apiConfig, filename, callback) { +async function remove(apiConfig, filename) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof filename, 'string'); - assert.strictEqual(typeof callback, 'function'); // Result: none - - callback(new BoxError(BoxError.NOT_IMPLEMENTED, 'remove is not implemented')); + throw new BoxError(BoxError.NOT_IMPLEMENTED, 'remove is not implemented'); } -function removeDir(apiConfig, pathPrefix) { +async function removeDir(apiConfig, pathPrefix, progressCallback) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof pathPrefix, 'string'); + assert.strictEqual(typeof progressCallback, 'function'); // Result: none - var events = new EventEmitter(); - process.nextTick(function () { events.emit('done', new BoxError(BoxError.NOT_IMPLEMENTED, 'removeDir is not implemented')); }); - return events; + throw new BoxError(BoxError.NOT_IMPLEMENTED, 'removeDir is not implemented'); } async function remount(apiConfig) { diff --git a/src/storage/noop.js b/src/storage/noop.js index b80c1eecd..acc08067b 100644 --- a/src/storage/noop.js +++ b/src/storage/noop.js @@ -104,25 +104,19 @@ function copy(apiConfig, oldFilePath, newFilePath) { return events; } -function remove(apiConfig, filename, callback) { +async function remove(apiConfig, filename) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof filename, 'string'); - assert.strictEqual(typeof callback, 'function'); - debug('remove: %s', filename); - - callback(null); + debug(`remove: ${filename}`); } -function removeDir(apiConfig, pathPrefix) { +function removeDir(apiConfig, pathPrefix, progressCallback) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof pathPrefix, 'string'); + assert.strictEqual(typeof progressCallback, 'function'); - debug('removeDir: %s', pathPrefix); - - var events = new EventEmitter(); - process.nextTick(function () { events.emit('done', null); }); - return events; + debug(`removeDir: ${pathPrefix}`); } async function remount(apiConfig) { diff --git a/src/storage/s3.js b/src/storage/s3.js index e59189951..183ce6ece 100644 --- a/src/storage/s3.js +++ b/src/storage/s3.js @@ -39,6 +39,7 @@ const assert = require('assert'), path = require('path'), S3BlockReadStream = require('s3-block-read-stream'), safe = require('safetydance'), + util = require('util'), _ = require('underscore'); let aws = AwsSdk; @@ -385,10 +386,9 @@ function copy(apiConfig, oldFilePath, newFilePath) { return events; } -function remove(apiConfig, filename, callback) { +async function remove(apiConfig, filename) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof filename, 'string'); - assert.strictEqual(typeof callback, 'function'); const credentials = getS3Config(apiConfig); @@ -402,57 +402,48 @@ function remove(apiConfig, filename, callback) { }; // deleteObjects does not return error if key is not found - s3.deleteObjects(deleteParams, function (error) { - if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Unable to remove ${deleteParams.Key}. error: ${error.message || error.code}`)); // DO sets 'code' - - callback(null); - }); + const [error] = await safe(s3.deleteObjects(deleteParams).promise()); + if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, `Unable to remove ${deleteParams.Key}. error: ${error.message}`); } -function removeDir(apiConfig, pathPrefix) { +async function removeDir(apiConfig, pathPrefix, progressCallback) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof pathPrefix, 'string'); + assert.strictEqual(typeof progressCallback, 'function'); - const events = new EventEmitter(); - let total = 0; const credentials = getS3Config(apiConfig); - const s3 = new aws.S3(credentials); + const listDirAsync = util.promisify(listDir); - listDir(apiConfig, pathPrefix, 1000, function listDirIterator(entries, done) { + let total = 0; + + await listDirAsync(apiConfig, pathPrefix, 1000, function listDirIterator(entries, done) { total += entries.length; const chunkSize = apiConfig.deleteConcurrency || (apiConfig.provider !== 'digitalocean-spaces' ? 1000 : 100); // throttle objects in each request - var chunks = chunk(entries, chunkSize); + const chunks = chunk(entries, chunkSize); - async.eachSeries(chunks, function deleteFiles(objects, iteratorCallback) { - var deleteParams = { + async.eachSeries(chunks, async function deleteFiles(objects) { + const deleteParams = { Bucket: apiConfig.bucket, Delete: { Objects: objects.map(function (o) { return { Key: o.fullPath }; }) } }; - events.emit('progress', `Removing ${objects.length} files from ${objects[0].fullPath} to ${objects[objects.length-1].fullPath}`); + progressCallback({ message: `Removing ${objects.length} files from ${objects[0].fullPath} to ${objects[objects.length-1].fullPath}` }); // deleteObjects does not return error if key is not found - s3.deleteObjects(deleteParams, function (error /*, deleteData */) { - if (error) { - events.emit('progress', `Unable to remove ${deleteParams.Key} ${error.message || error.code}`); - return iteratorCallback(new BoxError(BoxError.EXTERNAL_ERROR, `Unable to remove ${deleteParams.Key}. error: ${error.message || error.code}`)); // DO sets 'code' - } - - iteratorCallback(null); - }); + const [error] = await safe(s3.deleteObjects(deleteParams).promise()); + if (error) { + progressCallback({ message: `Unable to remove ${deleteParams.Key} ${error.message || error.code}` }); + throw new BoxError(BoxError.EXTERNAL_ERROR, `Unable to remove ${deleteParams.Key}. error: ${error.message}`); + } }, done); - }, function (error) { - events.emit('progress', `Removed ${total} files`); - - process.nextTick(() => events.emit('done', error)); }); - return events; + progressCallback({ message: `Removed ${total} files` }); } async function remount(apiConfig) {