diff --git a/migrations/20170928003352-backups-add-format.js b/migrations/20170928003352-backups-add-format.js new file mode 100644 index 000000000..fb0bcbbcd --- /dev/null +++ b/migrations/20170928003352-backups-add-format.js @@ -0,0 +1,15 @@ +'use strict'; + +exports.up = function(db, callback) { + db.runSql('ALTER TABLE backups ADD COLUMN format VARCHAR(16) DEFAULT "tgz"', function (error) { + if (error) console.error(error); + callback(error); + }); +}; + +exports.down = function(db, callback) { + db.runSql('ALTER TABLE backups DROP COLUMN format', function (error) { + if (error) console.error(error); + callback(error); + }); +}; diff --git a/src/backupdb.js b/src/backupdb.js index 566d68345..ef4617499 100644 --- a/src/backupdb.js +++ b/src/backupdb.js @@ -6,7 +6,7 @@ var assert = require('assert'), safe = require('safetydance'), util = require('util'); -var BACKUPS_FIELDS = [ 'id', 'creationTime', 'version', 'type', 'dependsOn', 'state', 'restoreConfigJson' ]; +var BACKUPS_FIELDS = [ 'id', 'creationTime', 'version', 'type', 'dependsOn', 'state', 'restoreConfigJson', 'format' ]; exports = module.exports = { add: add, @@ -47,12 +47,12 @@ function getByTypeAndStatePaged(type, state, page, perPage, callback) { database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups WHERE type = ? AND state = ? ORDER BY creationTime DESC LIMIT ?,?', [ type, state, (page-1)*perPage, perPage ], function (error, results) { - if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error)); + if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error)); - results.forEach(function (result) { postProcess(result); }); + results.forEach(function (result) { postProcess(result); }); - callback(null, results); - }); + callback(null, results); + }); } function getByTypePaged(type, page, perPage, callback) { @@ -63,12 +63,12 @@ function getByTypePaged(type, page, perPage, callback) { database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups WHERE type = ? ORDER BY creationTime DESC LIMIT ?,?', [ type, (page-1)*perPage, perPage ], function (error, results) { - if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error)); + if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error)); - results.forEach(function (result) { postProcess(result); }); + results.forEach(function (result) { postProcess(result); }); - callback(null, results); - }); + callback(null, results); + }); } function getByAppIdPaged(page, perPage, appId, callback) { @@ -80,12 +80,12 @@ function getByAppIdPaged(page, perPage, appId, callback) { // box versions (0.93.x and below) used to use appbackup_ prefix database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups WHERE type = ? AND state = ? AND id LIKE ? ORDER BY creationTime DESC LIMIT ?,?', [ exports.BACKUP_TYPE_APP, exports.BACKUP_STATE_NORMAL, '%app%\\_' + appId + '\\_%', (page-1)*perPage, perPage ], function (error, results) { - if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error)); + if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error)); - results.forEach(function (result) { postProcess(result); }); + results.forEach(function (result) { postProcess(result); }); - callback(null, results); - }); + callback(null, results); + }); } function get(id, callback) { @@ -94,13 +94,13 @@ function get(id, callback) { database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups WHERE id = ? ORDER BY creationTime DESC', [ id ], function (error, result) { - if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error)); - if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND)); + if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error)); + if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND)); - postProcess(result[0]); + postProcess(result[0]); - callback(null, result[0]); - }); + callback(null, result[0]); + }); } function add(backup, callback) { @@ -110,19 +110,20 @@ function add(backup, callback) { assert(backup.type === exports.BACKUP_TYPE_APP || backup.type === exports.BACKUP_TYPE_BOX); assert(util.isArray(backup.dependsOn)); assert.strictEqual(typeof backup.restoreConfig, 'object'); + assert.strictEqual(typeof backup.format, 'string'); assert.strictEqual(typeof callback, 'function'); var creationTime = backup.creationTime || new Date(); // allow tests to set the time var restoreConfig = backup.restoreConfig ? JSON.stringify(backup.restoreConfig) : ''; - database.query('INSERT INTO backups (id, version, type, creationTime, state, dependsOn, restoreConfigJson) VALUES (?, ?, ?, ?, ?, ?, ?)', - [ backup.id, backup.version, backup.type, creationTime, exports.BACKUP_STATE_NORMAL, backup.dependsOn.join(','), restoreConfig ], + database.query('INSERT INTO backups (id, version, type, creationTime, state, dependsOn, restoreConfigJson, format) VALUES (?, ?, ?, ?, ?, ?, ?, ?)', + [ backup.id, backup.version, backup.type, creationTime, exports.BACKUP_STATE_NORMAL, backup.dependsOn.join(','), restoreConfig, backup.format ], function (error) { - if (error && error.code === 'ER_DUP_ENTRY') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS)); - if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error)); + if (error && error.code === 'ER_DUP_ENTRY') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS)); + if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error)); - callback(null); - }); + callback(null); + }); } function update(id, backup, callback) { diff --git a/src/backups.js b/src/backups.js index 669053e83..65c590585 100644 --- a/src/backups.js +++ b/src/backups.js @@ -160,15 +160,16 @@ function getRestoreConfig(backupId, callback) { }); } -function getBackupFilePath(backupConfig, backupId, subpath) { +function getBackupFilePath(backupConfig, backupId, format) { assert.strictEqual(typeof backupConfig, 'object'); assert.strictEqual(typeof backupId, 'string'); + assert.strictEqual(typeof format, 'string'); - if (backupConfig.format === 'tgz') { + if (format === 'tgz') { const fileType = backupConfig.key ? '.tar.gz.enc' : '.tar.gz'; return path.join(backupConfig.prefix || backupConfig.backupFolder, backupId+fileType); } else { - return path.join(backupConfig.prefix || backupConfig.backupFolder, backupId, subpath || ''); + return path.join(backupConfig.prefix || backupConfig.backupFolder, backupId); } } @@ -216,16 +217,26 @@ function createTarPackStream(sourceDir, key) { } function sync(backupConfig, backupId, dataDir, callback) { + assert.strictEqual(typeof backupConfig, 'object'); + assert.strictEqual(typeof backupId, 'string'); + assert.strictEqual(typeof dataDir, 'string'); + assert.strictEqual(typeof callback, 'function'); + syncer.sync(dataDir, function processTask(task, iteratorCallback) { debug('processing task: %j', task); + var backupFilePath = path.join(getBackupFilePath(backupConfig, backupId, backupConfig.format), task.path); + if (task.operation === 'add') { safe.fs.writeFileSync(paths.BACKUP_RESULT_FILE, 'Adding ' + task.path); var stream = fs.createReadStream(path.join(dataDir, task.path)); stream.on('error', function () { return iteratorCallback(); }); // ignore error if file disappears - api(backupConfig.provider).upload(backupConfig, getBackupFilePath(backupConfig, backupId, task.path), stream, iteratorCallback); - } else if (task.operation === 'remove' || task.operation === 'removedir') { + api(backupConfig.provider).upload(backupConfig, backupFilePath, stream, iteratorCallback); + } else if (task.operation === 'remove') { safe.fs.writeFileSync(paths.BACKUP_RESULT_FILE, 'Removing ' + task.path); - api(backupConfig.provider).remove(backupConfig, getBackupFilePath(backupConfig, backupId, task.path), iteratorCallback); + api(backupConfig.provider).remove(backupConfig, backupFilePath, iteratorCallback); + } else if (task.operation === 'removedir') { + safe.fs.writeFileSync(paths.BACKUP_RESULT_FILE, 'Removing directory ' + task.path); + api(backupConfig.provider).removeDir(backupConfig, backupFilePath, iteratorCallback); } }, 10 /* concurrency */, function (error) { if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message)); @@ -247,8 +258,10 @@ function saveEmptyDirs(appDataDir, callback) { } // this function is called via backuptask (since it needs root to traverse app's directory) -function upload(backupId, dataDir, callback) { +function upload(backupId, format, dataDir, callback) { assert.strictEqual(typeof backupId, 'string'); + assert.strictEqual(typeof format, 'string'); + assert.strictEqual(typeof dataDir, 'string'); assert.strictEqual(typeof callback, 'function'); callback = once(callback); @@ -258,10 +271,10 @@ function upload(backupId, dataDir, callback) { settings.getBackupConfig(function (error, backupConfig) { if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error)); - if (backupConfig.format === 'tgz') { + if (format === 'tgz') { var tarStream = createTarPackStream(dataDir, backupConfig.key || null); tarStream.on('error', callback); // already returns BackupsError - api(backupConfig.provider).upload(backupConfig, getBackupFilePath(backupConfig, backupId), tarStream, callback); + api(backupConfig.provider).upload(backupConfig, getBackupFilePath(backupConfig, backupId, format), tarStream, callback); } else { async.series([ saveEmptyDirs.bind(null, dataDir), @@ -328,39 +341,41 @@ function createEmptyDirs(appDataDir, callback) { }, callback); } -function download(backupId, dataDir, callback) { +function download(backupId, format, dataDir, callback) { assert.strictEqual(typeof backupId, 'string'); + assert.strictEqual(typeof format, 'string'); assert.strictEqual(typeof dataDir, 'string'); assert.strictEqual(typeof callback, 'function'); - debug('Start download of id %s to %s', backupId, dataDir); + debug('Start download of id %s to %s (%s)', backupId, dataDir, format); settings.getBackupConfig(function (error, backupConfig) { if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error)); - if (backupConfig.format === 'tgz') { - api(backupConfig.provider).download(backupConfig, getBackupFilePath(backupConfig, backupId), function (error, sourceStream) { + if (format === 'tgz') { + api(backupConfig.provider).download(backupConfig, getBackupFilePath(backupConfig, backupId, format), function (error, sourceStream) { if (error) return callback(error); tarExtract(sourceStream, dataDir, backupConfig.key || null, callback); }); } else { async.series([ - api(backupConfig.provider).downloadDir.bind(null, backupConfig, getBackupFilePath(backupConfig, backupId), dataDir), + api(backupConfig.provider).downloadDir.bind(null, backupConfig, getBackupFilePath(backupConfig, backupId, format), dataDir), createEmptyDirs.bind(null, dataDir) ], callback); } }); } -function runBackupTask(backupId, dataDir, callback) { +function runBackupTask(backupId, format, dataDir, callback) { assert.strictEqual(typeof backupId, 'string'); + assert.strictEqual(typeof format, 'string'); assert.strictEqual(typeof dataDir, 'string'); assert.strictEqual(typeof callback, 'function'); var killTimerId = null, progressTimerId = null; - var cp = shell.sudo(`backup-${backupId}`, [ BACKUPTASK_CMD, backupId, dataDir ], { env: process.env, logFile: paths.BACKUP_LOG_FILE }, function (error) { + var cp = shell.sudo(`backup-${backupId}`, [ BACKUPTASK_CMD, backupId, format, dataDir ], { env: process.env, logFile: paths.BACKUP_LOG_FILE }, function (error) { clearTimeout(killTimerId); clearInterval(progressTimerId); @@ -434,12 +449,12 @@ function uploadBoxSnapshot(backupConfig, callback) { snapshotBox(function (error) { if (error) return callback(error); - runBackupTask('snapshot/box', paths.BOX_DATA_DIR, function (error) { + runBackupTask('snapshot/box', backupConfig.format, paths.BOX_DATA_DIR, function (error) { if (error) return callback(error); debug('uploadBoxSnapshot: time: %s secs', (new Date() - startTime)/1000); - setSnapshotInfo('box', { timestamp: new Date().toISOString() }, callback); + setSnapshotInfo('box', { timestamp: new Date().toISOString(), format: backupConfig.format }, callback); }); }); } @@ -458,10 +473,10 @@ function rotateBoxBackup(backupConfig, timestamp, appBackupIds, callback) { debug('rotateBoxBackup: rotating to id:%s', backupId); - backupdb.add({ id: backupId, version: config.version(), type: backupdb.BACKUP_TYPE_BOX, dependsOn: appBackupIds, restoreConfig: null }, function (error) { + backupdb.add({ id: backupId, version: config.version(), type: backupdb.BACKUP_TYPE_BOX, dependsOn: appBackupIds, restoreConfig: null, format: backupConfig.format }, function (error) { if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error)); - api(backupConfig.provider).copy(backupConfig, getBackupFilePath(backupConfig, 'snapshot/box'), getBackupFilePath(backupConfig, backupId), function (copyBackupError) { + api(backupConfig.provider).copy(backupConfig, getBackupFilePath(backupConfig, 'snapshot/box', backupConfig.format), getBackupFilePath(backupConfig, backupId, backupConfig.format), function (copyBackupError) { const state = copyBackupError ? backupdb.BACKUP_STATE_ERROR : backupdb.BACKUP_STATE_NORMAL; backupdb.update(backupId, { state: state }, function (error) { @@ -554,10 +569,10 @@ function rotateAppBackup(backupConfig, app, timestamp, callback) { debugApp(app, 'rotateAppBackup: rotating to id:%s', backupId); - backupdb.add({ id: backupId, version: manifest.version, type: backupdb.BACKUP_TYPE_APP, dependsOn: [ ], restoreConfig: restoreConfig }, function (error) { + backupdb.add({ id: backupId, version: manifest.version, type: backupdb.BACKUP_TYPE_APP, dependsOn: [ ], restoreConfig: restoreConfig, format: backupConfig.format }, function (error) { if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error)); - api(backupConfig.provider).copy(backupConfig, getBackupFilePath(backupConfig, `snapshot/app_${app.id}`), getBackupFilePath(backupConfig, backupId), function (copyBackupError) { + api(backupConfig.provider).copy(backupConfig, getBackupFilePath(backupConfig, `snapshot/app_${app.id}`, backupConfig.format), getBackupFilePath(backupConfig, backupId, backupConfig.format), function (copyBackupError) { const state = copyBackupError ? backupdb.BACKUP_STATE_ERROR : backupdb.BACKUP_STATE_NORMAL; debugApp(app, 'rotateAppBackup: successful id:%s', backupId); @@ -590,12 +605,12 @@ function uploadAppSnapshot(backupConfig, app, manifest, callback) { var backupId = util.format('snapshot/app_%s', app.id); var appDataDir = safe.fs.realpathSync(path.join(paths.APPS_DATA_DIR, app.id)); - runBackupTask(backupId, appDataDir, function (error) { + runBackupTask(backupId, backupConfig.format, appDataDir, function (error) { if (error) return callback(error); debugApp(app, 'uploadAppSnapshot: %s done time: %s secs', backupId, (new Date() - startTime)/1000); - setSnapshotInfo(app.id, { timestamp: new Date().toISOString(), restoreConfig: restoreConfig }, callback); + setSnapshotInfo(app.id, { timestamp: new Date().toISOString(), restoreConfig: restoreConfig, format: backupConfig.format }, callback); }); }); } @@ -733,13 +748,18 @@ function restoreApp(app, addonsToRestore, backupId, callback) { var startTime = new Date(); - async.series([ - download.bind(null, backupId, appDataDir), - addons.restoreAddons.bind(null, app, addonsToRestore) - ], function (error) { - debug('restoreApp: time: %s', (new Date() - startTime)/1000); + backupdb.get(backupId, function (error, result) { + if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new BackupsError(BackupsError.NOT_FOUND, error)); + if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error)); - callback(error); + async.series([ + download.bind(null, backupId, result.format, appDataDir), + addons.restoreAddons.bind(null, app, addonsToRestore) + ], function (error) { + debug('restoreApp: time: %s', (new Date() - startTime)/1000); + + callback(error); + }); }); } @@ -760,7 +780,9 @@ function cleanupAppBackups(backupConfig, referencedAppBackups, callback) { debug('cleanupAppBackups: removing %s', backup.id); - api(backupConfig.provider).remove(backupConfig, getBackupFilePath(backupConfig, backup.id), function (error) { + var removeFunc = backup.format ==='tgz' ? api(backupConfig.provider).remove : api(backupConfig.provider).removeDir; + + removeFunc(backupConfig, getBackupFilePath(backupConfig, backup.id, backup.format), function (error) { if (error) { debug('cleanupAppBackups: error removing backup %j : %s', backup, error.message); iteratorDone(); @@ -817,9 +839,12 @@ function cleanupBoxBackups(backupConfig, callback) { debug('cleanupBoxBackups: removing %s', backup.id); - var filePaths = [].concat(backup.id, backup.dependsOn).map(getBackupFilePath.bind(null, backupConfig)); + // TODO: assumes all backups have the same format + var filePaths = [].concat(backup.id, backup.dependsOn).map(function (id) { return getBackupFilePath(backupConfig, id, backup.format); }); - async.eachSeries(filePaths, api(backupConfig.provider).remove.bind(null, backupConfig), function (error) { + var removeFunc = backup.format ==='tgz' ? api(backupConfig.provider).remove : api(backupConfig.provider).removeDir; + + async.eachSeries(filePaths, removeFunc.bind(null, backupConfig), function (error) { if (error) { debug('cleanupBoxBackups: error removing backup %j : %s', backup, error.message); iteratorDone(); @@ -854,7 +879,8 @@ function cleanupSnapshots(backupConfig, callback) { apps.get(appId, function (error /*, app */) { if (!error || error.reason !== AppsError.NOT_FOUND) return iteratorDone(); - api(backupConfig.provider).remove(backupConfig, getBackupFilePath(backupConfig, `snapshot/app_${appId}`), function (/* ignoredError */) { + var removeFunc = info[appId].format ==='tgz' ? api(backupConfig.provider).remove : api(backupConfig.provider).removeDir; + removeFunc(backupConfig, getBackupFilePath(backupConfig, `snapshot/app_${appId}`, info[appId].format), function (/* ignoredError */) { setSnapshotInfo(appId, null); safe.fs.unlinkSync(path.join(paths.BACKUP_INFO_DIR, `${appId}.sync.cache`)); diff --git a/src/backuptask.js b/src/backuptask.js index 7c9fabe40..45a04545d 100755 --- a/src/backuptask.js +++ b/src/backuptask.js @@ -29,7 +29,8 @@ function initialize(callback) { // Main process starts here var backupId = process.argv[2]; -var dataDir = process.argv[3]; +var format = process.argv[3]; +var dataDir = process.argv[4]; debug(`Backing up ${dataDir} to ${backupId}`); @@ -42,7 +43,7 @@ initialize(function (error) { safe.fs.writeFileSync(paths.BACKUP_RESULT_FILE, ''); - backups.upload(backupId, dataDir, function resultHandler(error) { + backups.upload(backupId, format, dataDir, function resultHandler(error) { if (error) debug('completed with error', error); debug('completed'); diff --git a/src/storage/filesystem.js b/src/storage/filesystem.js index cfc556bac..e42cea07a 100644 --- a/src/storage/filesystem.js +++ b/src/storage/filesystem.js @@ -8,6 +8,7 @@ exports = module.exports = { copy: copy, remove: remove, + removeDir: removeDir, backupDone: backupDone, @@ -110,12 +111,24 @@ function copy(apiConfig, oldFilePath, newFilePath, callback) { }); } -function remove(apiConfig, pathPrefix, callback) { +function remove(apiConfig, filename, callback) { + assert.strictEqual(typeof apiConfig, 'object'); + assert.strictEqual(typeof filename, 'string'); + assert.strictEqual(typeof callback, 'function'); + + safe.fs.unlinkSync(filename); + + safe.fs.rmdirSync(path.dirname(filename)); // try to cleanup empty directories + + callback(); +} + +function removeDir(apiConfig, pathPrefix, callback) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof pathPrefix, 'string'); assert.strictEqual(typeof callback, 'function'); - shell.exec('remove', '/bin/rm', [ '-rf', pathPrefix ], { }, function (error) { + shell.exec('removeDir', '/bin/rm', [ '-rf', pathPrefix ], { }, function (error) { if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message)); safe.fs.rmdirSync(path.dirname(pathPrefix)); // try to cleanup empty directories diff --git a/src/storage/interface.js b/src/storage/interface.js index c4f2c6480..0be9390e9 100644 --- a/src/storage/interface.js +++ b/src/storage/interface.js @@ -13,6 +13,7 @@ exports = module.exports = { copy: copy, remove: remove, + removeDir: removeDir, backupDone: backupDone, @@ -61,7 +62,17 @@ function copy(apiConfig, oldFilePath, newFilePath, callback) { callback(new Error('not implemented')); } -function remove(apiConfig, pathPrefix, callback) { +function remove(apiConfig, filename, callback) { + assert.strictEqual(typeof apiConfig, 'object'); + assert.strictEqual(typeof filename, 'string'); + assert.strictEqual(typeof callback, 'function'); + + // Result: none + + callback(new Error('not implemented')); +} + +function removeDir(apiConfig, pathPrefix, callback) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof pathPrefix, 'string'); assert.strictEqual(typeof callback, 'function'); diff --git a/src/storage/noop.js b/src/storage/noop.js index 2515ba76b..8ea244a0e 100644 --- a/src/storage/noop.js +++ b/src/storage/noop.js @@ -7,6 +7,7 @@ exports = module.exports = { copy: copy, remove: remove, + removeDir: removeDir, backupDone: backupDone, @@ -59,12 +60,22 @@ function copy(apiConfig, oldFilePath, newFilePath, callback) { callback(); } -function remove(apiConfig, pathPrefix, callback) { +function remove(apiConfig, filename, callback) { + assert.strictEqual(typeof apiConfig, 'object'); + assert.strictEqual(typeof filename, 'string'); + assert.strictEqual(typeof callback, 'function'); + + debug('remove: %s', filename); + + callback(); +} + +function removeDir(apiConfig, pathPrefix, callback) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof pathPrefix, 'string'); assert.strictEqual(typeof callback, 'function'); - debug('remove: %s', pathPrefix); + debug('removeDir: %s', pathPrefix); callback(); } diff --git a/src/storage/s3.js b/src/storage/s3.js index 02cb73634..02b77c6fe 100644 --- a/src/storage/s3.js +++ b/src/storage/s3.js @@ -7,6 +7,7 @@ exports = module.exports = { copy: copy, remove: remove, + removeDir: removeDir, backupDone: backupDone, @@ -253,7 +254,32 @@ function copy(apiConfig, oldFilePath, newFilePath, callback) { }); } -function remove(apiConfig, pathPrefix, callback) { +function remove(apiConfig, filename, callback) { + assert.strictEqual(typeof apiConfig, 'object'); + assert.strictEqual(typeof filename, 'string'); + assert.strictEqual(typeof callback, 'function'); + + getBackupCredentials(apiConfig, function (error, credentials) { + if (error) return callback(error); + + var s3 = new AWS.S3(credentials); + + var deleteParams = { + Bucket: apiConfig.bucket, + Delete: { + Objects: [{ Key: filename }] + } + }; + + s3.deleteObjects(deleteParams, function (error) { + if (error) debug('remove: Unable to remove %s. Not fatal.', deleteParams.Key, error); + + callback(null); + }); + }); +} + +function removeDir(apiConfig, pathPrefix, callback) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof pathPrefix, 'string'); assert.strictEqual(typeof callback, 'function'); @@ -270,7 +296,7 @@ function remove(apiConfig, pathPrefix, callback) { async.forever(function listAndDelete(iteratorCallback) { s3.listObjectsV2(listParams, function (error, listData) { if (error) { - debug('remove: Failed to list %s. Not fatal.', error); + debug('removeDir: Failed to list %s. Not fatal.', error); return iteratorCallback(error); } @@ -283,10 +309,10 @@ function remove(apiConfig, pathPrefix, callback) { s3.deleteObjects(deleteParams, function (error, deleteData) { if (error) { - debug('remove: Unable to remove %s. Not fatal.', deleteParams.Key, error); + debug('removeDir: Unable to remove %s. Not fatal.', deleteParams.Key, error); return iteratorCallback(error); } - debug(': Deleted: %j Errors: %j', deleteData.Deleted, deleteData.Errors); + debug('removeDir: Deleted: %j Errors: %j', deleteData.Deleted, deleteData.Errors); listParams.Marker = listData.Contents[listData.Contents.length - 1].Key; // NextMarker is returned only with delimiter @@ -306,7 +332,7 @@ function testConfig(apiConfig, callback) { assert.strictEqual(typeof callback, 'function'); if (apiConfig.provider === 'caas') { - if (typeof apiConfig.token !== 'string') return callback(new BackupsError(BackupsError.BAD_FIELD, 'token must be a string')); + if (typeof apiConfig.token !== 'string') return callback(new BackupsError(BackupsError.BAD_FIELD, 'token must be a string')); } else { if (typeof apiConfig.accessKeyId !== 'string') return callback(new BackupsError(BackupsError.BAD_FIELD, 'accessKeyId must be a string')); if (typeof apiConfig.secretAccessKey !== 'string') return callback(new BackupsError(BackupsError.BAD_FIELD, 'secretAccessKey must be a string')); diff --git a/src/syncer.js b/src/syncer.js index 57a48fd8d..fa86c74cc 100644 --- a/src/syncer.js +++ b/src/syncer.js @@ -40,7 +40,7 @@ function sync(dir, taskProcessor, concurrency, callback) { var cacheFile = path.join(paths.BACKUP_INFO_DIR, path.basename(dir) + '.sync.cache'), newCacheFile = path.join(paths.BACKUP_INFO_DIR, path.basename(dir) + '.sync.cache.new'); - if (!safe.fs.existsSync(cacheFile)) { // if cache is missing, start out empty + if (!safe.fs.existsSync(cacheFile)) { // if cache is missing, start out empty. TODO: do a remote listDir and rebuild delQueue.push({ operation: 'removedir', path: '' }); } @@ -50,6 +50,7 @@ function sync(dir, taskProcessor, concurrency, callback) { if (newCacheFd === -1) return callback(new Error('Error opening new cache file: ' + safe.error.message)); function advanceCache(entryPath) { + // TODO: detect and issue removedir for (; curCacheIndex !== cache.length && (entryPath === '' || cache[curCacheIndex].path < entryPath); ++curCacheIndex) { delQueue.push({ operation: 'remove', path: cache[curCacheIndex].path }); } diff --git a/src/test/backups-test.js b/src/test/backups-test.js index 57c9c71dc..7d8dde73e 100644 --- a/src/test/backups-test.js +++ b/src/test/backups-test.js @@ -7,7 +7,6 @@ 'use strict'; var async = require('async'), - appdb = require('../appdb.js'), backupdb = require('../backupdb.js'), backups = require('../backups.js'), database = require('../database'), @@ -46,7 +45,8 @@ describe('backups', function () { version: '1.0.0', type: backupdb.BACKUP_TYPE_BOX, dependsOn: [ 'backup-app-00', 'backup-app-01' ], - restoreConfig: null + restoreConfig: null, + format: 'tgz' }; var BACKUP_0_APP_0 = { @@ -54,7 +54,8 @@ describe('backups', function () { version: '1.0.0', type: backupdb.BACKUP_TYPE_APP, dependsOn: [], - restoreConfig: null + restoreConfig: null, + format: 'tgz' }; var BACKUP_0_APP_1 = { @@ -62,7 +63,8 @@ describe('backups', function () { version: '1.0.0', type: backupdb.BACKUP_TYPE_APP, dependsOn: [], - restoreConfig: null + restoreConfig: null, + format: 'tgz' }; var BACKUP_1 = { @@ -70,7 +72,8 @@ describe('backups', function () { version: '1.0.0', type: backupdb.BACKUP_TYPE_BOX, dependsOn: [ 'backup-app-10', 'backup-app-11' ], - restoreConfig: null + restoreConfig: null, + format: 'tgz' }; var BACKUP_1_APP_0 = { @@ -78,7 +81,8 @@ describe('backups', function () { version: '1.0.0', type: backupdb.BACKUP_TYPE_APP, dependsOn: [], - restoreConfig: null + restoreConfig: null, + format: 'tgz' }; var BACKUP_1_APP_1 = { @@ -86,7 +90,8 @@ describe('backups', function () { version: '1.0.0', type: backupdb.BACKUP_TYPE_APP, dependsOn: [], - restoreConfig: null + restoreConfig: null, + format: 'tgz' }; it('succeeds without backups', function (done) { diff --git a/src/test/database-test.js b/src/test/database-test.js index 58e9e98f0..009666a70 100644 --- a/src/test/database-test.js +++ b/src/test/database-test.js @@ -1025,7 +1025,8 @@ describe('database', function () { version: '1.0.0', type: backupdb.BACKUP_TYPE_BOX, dependsOn: [ 'dep1' ], - restoreConfig: null + restoreConfig: null, + format: 'tgz' }; backupdb.add(backup, function (error) { @@ -1090,7 +1091,8 @@ describe('database', function () { version: '1.0.0', type: backupdb.BACKUP_TYPE_APP, dependsOn: [ ], - restoreConfig: { manifest: { foo: 'bar' } } + restoreConfig: { manifest: { foo: 'bar' } }, + format: 'tgz' }; backupdb.add(backup, function (error) { diff --git a/src/test/storage-test.js b/src/test/storage-test.js index 2367ceed5..7bb78b335 100644 --- a/src/test/storage-test.js +++ b/src/test/storage-test.js @@ -115,8 +115,8 @@ describe('Storage', function () { gSourceFolder = path.join(__dirname, 'storage'); gDestinationFolder = path.join(gTmpFolder, 'destination/'); - gBackupId_1 = backups._getBackupFilePath(gBackupConfig, 'someprefix/one'); - gBackupId_2 = backups._getBackupFilePath(gBackupConfig, 'someprefix/two'); + gBackupId_1 = backups._getBackupFilePath(gBackupConfig, 'someprefix/one', gBackupConfig.format); + gBackupId_2 = backups._getBackupFilePath(gBackupConfig, 'someprefix/two', gBackupConfig.format); done(); });