diff --git a/src/backups.js b/src/backups.js index ff92cb843..6abdf4323 100644 --- a/src/backups.js +++ b/src/backups.js @@ -234,6 +234,19 @@ function sync(backupConfig, backupId, dataDir, callback) { debug('sync: processing task: %j', task); var backupFilePath = path.join(getBackupFilePath(backupConfig, backupId, backupConfig.format), task.path); + if (task.operation === 'removedir') { + safe.fs.writeFileSync(paths.BACKUP_RESULT_FILE, `Removing directory ${task.path}`); + return api(backupConfig.provider).removeDir(backupConfig, backupFilePath) + .on('progress', function (detail) { + debug(`sync: ${detail}`); + safe.fs.writeFileSync(paths.BACKUP_RESULT_FILE, detail); + }) + .on('done', iteratorCallback); + } else if (task.operation === 'remove') { + safe.fs.writeFileSync(paths.BACKUP_RESULT_FILE, `Removing ${task.path}`); + return api(backupConfig.provider).remove(backupConfig, backupFilePath, iteratorCallback); + } + var retryCount = 0; async.retry({ times: 5, interval: 20000 }, function (retryCallback) { ++retryCount; @@ -243,16 +256,6 @@ function sync(backupConfig, backupId, dataDir, callback) { var stream = fs.createReadStream(path.join(dataDir, task.path)); stream.on('error', function () { return retryCallback(); }); // ignore error if file disappears api(backupConfig.provider).upload(backupConfig, backupFilePath, stream, retryCallback); - } else if (task.operation === 'remove') { - safe.fs.writeFileSync(paths.BACKUP_RESULT_FILE, `Removing ${task.path}`); - api(backupConfig.provider).remove(backupConfig, backupFilePath, retryCallback); - } else if (task.operation === 'removedir') { - var events = api(backupConfig.provider).removeDir(backupConfig, backupFilePath); - events.on('progress', function (detail) { - debug(`sync: ${detail}`); - safe.fs.writeFileSync(paths.BACKUP_RESULT_FILE, detail); - }); - events.on('done', retryCallback); } }, iteratorCallback); }, 10 /* concurrency */, function (error) { @@ -289,7 +292,7 @@ function upload(backupId, format, dataDir, callback) { if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error)); if (format === 'tgz') { - async.retry({ times: 3, interval: 10000 }, function (retryCallback) { + async.retry({ times: 5, interval: 20000 }, function (retryCallback) { var tarStream = createTarPackStream(dataDir, backupConfig.key || null); tarStream.on('error', retryCallback); // already returns BackupsError @@ -541,11 +544,9 @@ function rotateBoxBackup(backupConfig, timestamp, appBackupIds, callback) { backupdb.add({ id: backupId, version: config.version(), type: backupdb.BACKUP_TYPE_BOX, dependsOn: appBackupIds, restoreConfig: null, format: format }, function (error) { if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error)); - async.retry({ times: 3, interval: 10000 }, function (retryCallback) { - var copy = api(backupConfig.provider).copy(backupConfig, getBackupFilePath(backupConfig, 'snapshot/box', format), getBackupFilePath(backupConfig, backupId, format)); - copy.on('progress', log); - copy.on('done', retryCallback); - }, function (copyBackupError) { + var copy = api(backupConfig.provider).copy(backupConfig, getBackupFilePath(backupConfig, 'snapshot/box', format), getBackupFilePath(backupConfig, backupId, format)); + copy.on('progress', log); + copy.on('done', function (copyBackupError) { const state = copyBackupError ? backupdb.BACKUP_STATE_ERROR : backupdb.BACKUP_STATE_NORMAL; backupdb.update(backupId, { state: state }, function (error) { @@ -644,11 +645,9 @@ function rotateAppBackup(backupConfig, app, timestamp, callback) { backupdb.add({ id: backupId, version: manifest.version, type: backupdb.BACKUP_TYPE_APP, dependsOn: [ ], restoreConfig: restoreConfig, format: format }, function (error) { if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error)); - async.retry({ times: 3, interval: 10000 }, function (retryCallback) { - var copy = api(backupConfig.provider).copy(backupConfig, getBackupFilePath(backupConfig, `snapshot/app_${app.id}`, format), getBackupFilePath(backupConfig, backupId, format)); - copy.on('progress', log); - copy.on('done', retryCallback); - }, function (copyBackupError) { + var copy = api(backupConfig.provider).copy(backupConfig, getBackupFilePath(backupConfig, `snapshot/app_${app.id}`, format), getBackupFilePath(backupConfig, backupId, format)); + copy.on('progress', log); + copy.on('done', function (copyBackupError) { const state = copyBackupError ? backupdb.BACKUP_STATE_ERROR : backupdb.BACKUP_STATE_NORMAL; log(`Rotated app backup of ${app.id} successfully to id ${backupId}`); @@ -716,6 +715,7 @@ function backupApp(app, manifest, callback) { assert.strictEqual(typeof callback, 'function'); const timestamp = (new Date()).toISOString().replace(/[T.]/g, '-').replace(/[:Z]/g,''); + safe.fs.unlinkSync(paths.BACKUP_LOG_FILE); // start fresh log file progress.set(progress.BACKUP, 10, 'Backing up ' + (app.altDomain || config.appFqdn(app.location))); @@ -733,6 +733,7 @@ function backupBoxAndApps(auditSource, callback) { callback = callback || NOOP_CALLBACK; var timestamp = (new Date()).toISOString().replace(/[T.]/g, '-').replace(/[:Z]/g,''); + safe.fs.unlinkSync(paths.BACKUP_LOG_FILE); // start fresh log file eventlog.add(eventlog.ACTION_BACKUP_START, auditSource, { }); diff --git a/src/storage/s3.js b/src/storage/s3.js index 08e4179d6..2eb245d64 100644 --- a/src/storage/s3.js +++ b/src/storage/s3.js @@ -48,7 +48,7 @@ function mockRestore() { var gCachedCaasCredentials = { issueDate: null, credentials: null }; -function getCaasCredentials(apiConfig, callback) { +function getCaasConfig(apiConfig, callback) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof callback, 'function'); assert(apiConfig.token); @@ -70,7 +70,11 @@ function getCaasCredentials(apiConfig, callback) { accessKeyId: result.body.credentials.AccessKeyId, secretAccessKey: result.body.credentials.SecretAccessKey, sessionToken: result.body.credentials.SessionToken, - region: apiConfig.region || 'us-east-1' + region: apiConfig.region || 'us-east-1', + httpOptions: { + timeout: 60000 + }, + maxRetries: 5 }; if (apiConfig.endpoint) credentials.endpoint = new AWS.Endpoint(apiConfig.endpoint); @@ -84,18 +88,22 @@ function getCaasCredentials(apiConfig, callback) { }); } -function getBackupCredentials(apiConfig, callback) { +function getS3Config(apiConfig, callback) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof callback, 'function'); - if (apiConfig.provider === 'caas') return getCaasCredentials(apiConfig, callback); + if (apiConfig.provider === 'caas') return getCaasConfig(apiConfig, callback); var credentials = { signatureVersion: apiConfig.signatureVersion || 'v4', s3ForcePathStyle: true, // Force use path-style url (http://endpoint/bucket/path) instead of host-style (http://bucket.endpoint/path) accessKeyId: apiConfig.accessKeyId, secretAccessKey: apiConfig.secretAccessKey, - region: apiConfig.region || 'us-east-1' + region: apiConfig.region || 'us-east-1', + httpOptions: { + timeout: 60000 + }, + maxRetries: 5 }; if (apiConfig.endpoint) credentials.endpoint = apiConfig.endpoint; @@ -124,7 +132,7 @@ function upload(apiConfig, backupFilePath, sourceStream, callback) { callback(null); } - getBackupCredentials(apiConfig, function (error, credentials) { + getS3Config(apiConfig, function (error, credentials) { if (error) return callback(error); var params = { @@ -154,7 +162,7 @@ function download(apiConfig, backupFilePath, callback) { assert.strictEqual(typeof backupFilePath, 'string'); assert.strictEqual(typeof callback, 'function'); - getBackupCredentials(apiConfig, function (error, credentials) { + getS3Config(apiConfig, function (error, credentials) { if (error) return callback(error); var params = { @@ -183,7 +191,7 @@ function download(apiConfig, backupFilePath, callback) { } function listDir(apiConfig, backupFilePath, options, iteratorCallback, callback) { - getBackupCredentials(apiConfig, function (error, credentials) { + getS3Config(apiConfig, function (error, credentials) { if (error) return callback(error); var s3 = new AWS.S3(credentials); @@ -368,7 +376,7 @@ function remove(apiConfig, filename, callback) { assert.strictEqual(typeof filename, 'string'); assert.strictEqual(typeof callback, 'function'); - getBackupCredentials(apiConfig, function (error, credentials) { + getS3Config(apiConfig, function (error, credentials) { if (error) return callback(error); var s3 = new AWS.S3(credentials); @@ -441,7 +449,7 @@ function testConfig(apiConfig, callback) { if ('endpoint' in apiConfig && typeof apiConfig.endpoint !== 'string') return callback(new BackupsError(BackupsError.BAD_FIELD, 'endpoint must be a string')); // attempt to upload and delete a file with new credentials - getBackupCredentials(apiConfig, function (error, credentials) { + getS3Config(apiConfig, function (error, credentials) { if (error) return callback(error); var params = {