jjkiers/cloudron-box
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Add format to backupsdb

Browse Source 
Call remove/removeDir based on the format
This commit is contained in:
Girish Ramakrishnan
2017-09-27 17:34:49 -07:00
parent aa8ce2c62e
commit a34180c27b
12 changed files with 194 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/backupdb.js
+25 -24
View File
@@ -6,7 +6,7 @@ var assert = require('assert'),
safe = require('safetydance'),
util = require('util');
var BACKUPS_FIELDS = [ 'id', 'creationTime', 'version', 'type', 'dependsOn', 'state', 'restoreConfigJson' ];
var BACKUPS_FIELDS = [ 'id', 'creationTime', 'version', 'type', 'dependsOn', 'state', 'restoreConfigJson', 'format' ];
exports = module.exports = {
add: add,
@@ -47,12 +47,12 @@ function getByTypeAndStatePaged(type, state, page, perPage, callback) {
database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups WHERE type = ? AND state = ? ORDER BY creationTime DESC LIMIT ?,?',
[ type, state, (page-1)*perPage, perPage ], function (error, results) {
if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
results.forEach(function (result) { postProcess(result); });
results.forEach(function (result) { postProcess(result); });
callback(null, results);
});
callback(null, results);
});
}
function getByTypePaged(type, page, perPage, callback) {
@@ -63,12 +63,12 @@ function getByTypePaged(type, page, perPage, callback) {
database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups WHERE type = ? ORDER BY creationTime DESC LIMIT ?,?',
[ type, (page-1)*perPage, perPage ], function (error, results) {
if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
results.forEach(function (result) { postProcess(result); });
results.forEach(function (result) { postProcess(result); });
callback(null, results);
});
callback(null, results);
});
}
function getByAppIdPaged(page, perPage, appId, callback) {
@@ -80,12 +80,12 @@ function getByAppIdPaged(page, perPage, appId, callback) {
// box versions (0.93.x and below) used to use appbackup_ prefix
database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups WHERE type = ? AND state = ? AND id LIKE ? ORDER BY creationTime DESC LIMIT ?,?',
[ exports.BACKUP_TYPE_APP, exports.BACKUP_STATE_NORMAL, '%app%\\_' + appId + '\\_%', (page-1)*perPage, perPage ], function (error, results) {
if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
results.forEach(function (result) { postProcess(result); });
results.forEach(function (result) { postProcess(result); });
callback(null, results);
});
callback(null, results);
});
}
function get(id, callback) {
@@ -94,13 +94,13 @@ function get(id, callback) {
database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups WHERE id = ? ORDER BY creationTime DESC',
[ id ], function (error, result) {
if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
postProcess(result[0]);
postProcess(result[0]);
callback(null, result[0]);
});
callback(null, result[0]);
});
}
function add(backup, callback) {
@@ -110,19 +110,20 @@ function add(backup, callback) {
assert(backup.type === exports.BACKUP_TYPE_APP || backup.type === exports.BACKUP_TYPE_BOX);
assert(util.isArray(backup.dependsOn));
assert.strictEqual(typeof backup.restoreConfig, 'object');
assert.strictEqual(typeof backup.format, 'string');
assert.strictEqual(typeof callback, 'function');
var creationTime = backup.creationTime || new Date(); // allow tests to set the time
var restoreConfig = backup.restoreConfig ? JSON.stringify(backup.restoreConfig) : '';
database.query('INSERT INTO backups (id, version, type, creationTime, state, dependsOn, restoreConfigJson) VALUES (?, ?, ?, ?, ?, ?, ?)',
[ backup.id, backup.version, backup.type, creationTime, exports.BACKUP_STATE_NORMAL, backup.dependsOn.join(','), restoreConfig ],
database.query('INSERT INTO backups (id, version, type, creationTime, state, dependsOn, restoreConfigJson, format) VALUES (?, ?, ?, ?, ?, ?, ?, ?)',
[ backup.id, backup.version, backup.type, creationTime, exports.BACKUP_STATE_NORMAL, backup.dependsOn.join(','), restoreConfig, backup.format ],
function (error) {
if (error && error.code === 'ER_DUP_ENTRY') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS));
if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
if (error && error.code === 'ER_DUP_ENTRY') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS));
if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
callback(null);
});
callback(null);
});
}
function update(id, backup, callback) {
src/backups.js
+60 -34
View File
@@ -160,15 +160,16 @@ function getRestoreConfig(backupId, callback) {
});
}
function getBackupFilePath(backupConfig, backupId, subpath) {
function getBackupFilePath(backupConfig, backupId, format) {
assert.strictEqual(typeof backupConfig, 'object');
assert.strictEqual(typeof backupId, 'string');
assert.strictEqual(typeof format, 'string');
if (backupConfig.format === 'tgz') {
if (format === 'tgz') {
const fileType = backupConfig.key ? '.tar.gz.enc' : '.tar.gz';
return path.join(backupConfig.prefix || backupConfig.backupFolder, backupId+fileType);
} else {
return path.join(backupConfig.prefix || backupConfig.backupFolder, backupId, subpath || '');
return path.join(backupConfig.prefix || backupConfig.backupFolder, backupId);
}
}
@@ -216,16 +217,26 @@ function createTarPackStream(sourceDir, key) {
}
function sync(backupConfig, backupId, dataDir, callback) {
assert.strictEqual(typeof backupConfig, 'object');
assert.strictEqual(typeof backupId, 'string');
assert.strictEqual(typeof dataDir, 'string');
assert.strictEqual(typeof callback, 'function');
syncer.sync(dataDir, function processTask(task, iteratorCallback) {
debug('processing task: %j', task);
var backupFilePath = path.join(getBackupFilePath(backupConfig, backupId, backupConfig.format), task.path);
if (task.operation === 'add') {
safe.fs.writeFileSync(paths.BACKUP_RESULT_FILE, 'Adding ' + task.path);
var stream = fs.createReadStream(path.join(dataDir, task.path));
stream.on('error', function () { return iteratorCallback(); }); // ignore error if file disappears
api(backupConfig.provider).upload(backupConfig, getBackupFilePath(backupConfig, backupId, task.path), stream, iteratorCallback);
} else if (task.operation === 'remove' || task.operation === 'removedir') {
api(backupConfig.provider).upload(backupConfig, backupFilePath, stream, iteratorCallback);
} else if (task.operation === 'remove') {
safe.fs.writeFileSync(paths.BACKUP_RESULT_FILE, 'Removing ' + task.path);
api(backupConfig.provider).remove(backupConfig, getBackupFilePath(backupConfig, backupId, task.path), iteratorCallback);
api(backupConfig.provider).remove(backupConfig, backupFilePath, iteratorCallback);
} else if (task.operation === 'removedir') {
safe.fs.writeFileSync(paths.BACKUP_RESULT_FILE, 'Removing directory ' + task.path);
api(backupConfig.provider).removeDir(backupConfig, backupFilePath, iteratorCallback);
}
}, 10 /* concurrency */, function (error) {
if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message));
@@ -247,8 +258,10 @@ function saveEmptyDirs(appDataDir, callback) {
}
// this function is called via backuptask (since it needs root to traverse app's directory)
function upload(backupId, dataDir, callback) {
function upload(backupId, format, dataDir, callback) {
assert.strictEqual(typeof backupId, 'string');
assert.strictEqual(typeof format, 'string');
assert.strictEqual(typeof dataDir, 'string');
assert.strictEqual(typeof callback, 'function');
callback = once(callback);
@@ -258,10 +271,10 @@ function upload(backupId, dataDir, callback) {
settings.getBackupConfig(function (error, backupConfig) {
if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
if (backupConfig.format === 'tgz') {
if (format === 'tgz') {
var tarStream = createTarPackStream(dataDir, backupConfig.key || null);
tarStream.on('error', callback); // already returns BackupsError
api(backupConfig.provider).upload(backupConfig, getBackupFilePath(backupConfig, backupId), tarStream, callback);
api(backupConfig.provider).upload(backupConfig, getBackupFilePath(backupConfig, backupId, format), tarStream, callback);
} else {
async.series([
saveEmptyDirs.bind(null, dataDir),
@@ -328,39 +341,41 @@ function createEmptyDirs(appDataDir, callback) {
}, callback);
}
function download(backupId, dataDir, callback) {
function download(backupId, format, dataDir, callback) {
assert.strictEqual(typeof backupId, 'string');
assert.strictEqual(typeof format, 'string');
assert.strictEqual(typeof dataDir, 'string');
assert.strictEqual(typeof callback, 'function');
debug('Start download of id %s to %s', backupId, dataDir);
debug('Start download of id %s to %s (%s)', backupId, dataDir, format);
settings.getBackupConfig(function (error, backupConfig) {
if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
if (backupConfig.format === 'tgz') {
api(backupConfig.provider).download(backupConfig, getBackupFilePath(backupConfig, backupId), function (error, sourceStream) {
if (format === 'tgz') {
api(backupConfig.provider).download(backupConfig, getBackupFilePath(backupConfig, backupId, format), function (error, sourceStream) {
if (error) return callback(error);
tarExtract(sourceStream, dataDir, backupConfig.key || null, callback);
});
} else {
async.series([
api(backupConfig.provider).downloadDir.bind(null, backupConfig, getBackupFilePath(backupConfig, backupId), dataDir),
api(backupConfig.provider).downloadDir.bind(null, backupConfig, getBackupFilePath(backupConfig, backupId, format), dataDir),
createEmptyDirs.bind(null, dataDir)
], callback);
}
});
}
function runBackupTask(backupId, dataDir, callback) {
function runBackupTask(backupId, format, dataDir, callback) {
assert.strictEqual(typeof backupId, 'string');
assert.strictEqual(typeof format, 'string');
assert.strictEqual(typeof dataDir, 'string');
assert.strictEqual(typeof callback, 'function');
var killTimerId = null, progressTimerId = null;
var cp = shell.sudo(`backup-${backupId}`, [ BACKUPTASK_CMD, backupId, dataDir ], { env: process.env, logFile: paths.BACKUP_LOG_FILE }, function (error) {
var cp = shell.sudo(`backup-${backupId}`, [ BACKUPTASK_CMD, backupId, format, dataDir ], { env: process.env, logFile: paths.BACKUP_LOG_FILE }, function (error) {
clearTimeout(killTimerId);
clearInterval(progressTimerId);
@@ -434,12 +449,12 @@ function uploadBoxSnapshot(backupConfig, callback) {
snapshotBox(function (error) {
if (error) return callback(error);
runBackupTask('snapshot/box', paths.BOX_DATA_DIR, function (error) {
runBackupTask('snapshot/box', backupConfig.format, paths.BOX_DATA_DIR, function (error) {
if (error) return callback(error);
debug('uploadBoxSnapshot: time: %s secs', (new Date() - startTime)/1000);
setSnapshotInfo('box', { timestamp: new Date().toISOString() }, callback);
setSnapshotInfo('box', { timestamp: new Date().toISOString(), format: backupConfig.format }, callback);
});
});
}
@@ -458,10 +473,10 @@ function rotateBoxBackup(backupConfig, timestamp, appBackupIds, callback) {
debug('rotateBoxBackup: rotating to id:%s', backupId);
backupdb.add({ id: backupId, version: config.version(), type: backupdb.BACKUP_TYPE_BOX, dependsOn: appBackupIds, restoreConfig: null }, function (error) {
backupdb.add({ id: backupId, version: config.version(), type: backupdb.BACKUP_TYPE_BOX, dependsOn: appBackupIds, restoreConfig: null, format: backupConfig.format }, function (error) {
if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
api(backupConfig.provider).copy(backupConfig, getBackupFilePath(backupConfig, 'snapshot/box'), getBackupFilePath(backupConfig, backupId), function (copyBackupError) {
api(backupConfig.provider).copy(backupConfig, getBackupFilePath(backupConfig, 'snapshot/box', backupConfig.format), getBackupFilePath(backupConfig, backupId, backupConfig.format), function (copyBackupError) {
const state = copyBackupError ? backupdb.BACKUP_STATE_ERROR : backupdb.BACKUP_STATE_NORMAL;
backupdb.update(backupId, { state: state }, function (error) {
@@ -554,10 +569,10 @@ function rotateAppBackup(backupConfig, app, timestamp, callback) {
debugApp(app, 'rotateAppBackup: rotating to id:%s', backupId);
backupdb.add({ id: backupId, version: manifest.version, type: backupdb.BACKUP_TYPE_APP, dependsOn: [ ], restoreConfig: restoreConfig }, function (error) {
backupdb.add({ id: backupId, version: manifest.version, type: backupdb.BACKUP_TYPE_APP, dependsOn: [ ], restoreConfig: restoreConfig, format: backupConfig.format }, function (error) {
if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
api(backupConfig.provider).copy(backupConfig, getBackupFilePath(backupConfig, `snapshot/app_${app.id}`), getBackupFilePath(backupConfig, backupId), function (copyBackupError) {
api(backupConfig.provider).copy(backupConfig, getBackupFilePath(backupConfig, `snapshot/app_${app.id}`, backupConfig.format), getBackupFilePath(backupConfig, backupId, backupConfig.format), function (copyBackupError) {
const state = copyBackupError ? backupdb.BACKUP_STATE_ERROR : backupdb.BACKUP_STATE_NORMAL;
debugApp(app, 'rotateAppBackup: successful id:%s', backupId);
@@ -590,12 +605,12 @@ function uploadAppSnapshot(backupConfig, app, manifest, callback) {
var backupId = util.format('snapshot/app_%s', app.id);
var appDataDir = safe.fs.realpathSync(path.join(paths.APPS_DATA_DIR, app.id));
runBackupTask(backupId, appDataDir, function (error) {
runBackupTask(backupId, backupConfig.format, appDataDir, function (error) {
if (error) return callback(error);
debugApp(app, 'uploadAppSnapshot: %s done time: %s secs', backupId, (new Date() - startTime)/1000);
setSnapshotInfo(app.id, { timestamp: new Date().toISOString(), restoreConfig: restoreConfig }, callback);
setSnapshotInfo(app.id, { timestamp: new Date().toISOString(), restoreConfig: restoreConfig, format: backupConfig.format }, callback);
});
});
}
@@ -733,13 +748,18 @@ function restoreApp(app, addonsToRestore, backupId, callback) {
var startTime = new Date();
async.series([
download.bind(null, backupId, appDataDir),
addons.restoreAddons.bind(null, app, addonsToRestore)
], function (error) {
debug('restoreApp: time: %s', (new Date() - startTime)/1000);
backupdb.get(backupId, function (error, result) {
if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new BackupsError(BackupsError.NOT_FOUND, error));
if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
callback(error);
async.series([
download.bind(null, backupId, result.format, appDataDir),
addons.restoreAddons.bind(null, app, addonsToRestore)
], function (error) {
debug('restoreApp: time: %s', (new Date() - startTime)/1000);
callback(error);
});
});
}
@@ -760,7 +780,9 @@ function cleanupAppBackups(backupConfig, referencedAppBackups, callback) {
debug('cleanupAppBackups: removing %s', backup.id);
api(backupConfig.provider).remove(backupConfig, getBackupFilePath(backupConfig, backup.id), function (error) {
var removeFunc = backup.format ==='tgz' ? api(backupConfig.provider).remove : api(backupConfig.provider).removeDir;
removeFunc(backupConfig, getBackupFilePath(backupConfig, backup.id, backup.format), function (error) {
if (error) {
debug('cleanupAppBackups: error removing backup %j : %s', backup, error.message);
iteratorDone();
@@ -817,9 +839,12 @@ function cleanupBoxBackups(backupConfig, callback) {
debug('cleanupBoxBackups: removing %s', backup.id);
var filePaths = [].concat(backup.id, backup.dependsOn).map(getBackupFilePath.bind(null, backupConfig));
// TODO: assumes all backups have the same format
var filePaths = [].concat(backup.id, backup.dependsOn).map(function (id) { return getBackupFilePath(backupConfig, id, backup.format); });
async.eachSeries(filePaths, api(backupConfig.provider).remove.bind(null, backupConfig), function (error) {
var removeFunc = backup.format ==='tgz' ? api(backupConfig.provider).remove : api(backupConfig.provider).removeDir;
async.eachSeries(filePaths, removeFunc.bind(null, backupConfig), function (error) {
if (error) {
debug('cleanupBoxBackups: error removing backup %j : %s', backup, error.message);
iteratorDone();
@@ -854,7 +879,8 @@ function cleanupSnapshots(backupConfig, callback) {
apps.get(appId, function (error /*, app */) {
if (!error || error.reason !== AppsError.NOT_FOUND) return iteratorDone();
api(backupConfig.provider).remove(backupConfig, getBackupFilePath(backupConfig, `snapshot/app_${appId}`), function (/* ignoredError */) {
var removeFunc = info[appId].format ==='tgz' ? api(backupConfig.provider).remove : api(backupConfig.provider).removeDir;
removeFunc(backupConfig, getBackupFilePath(backupConfig, `snapshot/app_${appId}`, info[appId].format), function (/* ignoredError */) {
setSnapshotInfo(appId, null);
safe.fs.unlinkSync(path.join(paths.BACKUP_INFO_DIR, `${appId}.sync.cache`));
src/backuptask.js
+3 -2
View File
@@ -29,7 +29,8 @@ function initialize(callback) {
// Main process starts here
var backupId = process.argv[2];
var dataDir = process.argv[3];
var format = process.argv[3];
var dataDir = process.argv[4];
debug(`Backing up ${dataDir} to ${backupId}`);
@@ -42,7 +43,7 @@ initialize(function (error) {
safe.fs.writeFileSync(paths.BACKUP_RESULT_FILE, '');
backups.upload(backupId, dataDir, function resultHandler(error) {
backups.upload(backupId, format, dataDir, function resultHandler(error) {
if (error) debug('completed with error', error);
debug('completed');
src/storage/filesystem.js
+15 -2
View File
@@ -8,6 +8,7 @@ exports = module.exports = {
copy: copy,
remove: remove,
removeDir: removeDir,
backupDone: backupDone,
@@ -110,12 +111,24 @@ function copy(apiConfig, oldFilePath, newFilePath, callback) {
});
}
function remove(apiConfig, pathPrefix, callback) {
function remove(apiConfig, filename, callback) {
assert.strictEqual(typeof apiConfig, 'object');
assert.strictEqual(typeof filename, 'string');
assert.strictEqual(typeof callback, 'function');
safe.fs.unlinkSync(filename);
safe.fs.rmdirSync(path.dirname(filename)); // try to cleanup empty directories
callback();
}
function removeDir(apiConfig, pathPrefix, callback) {
assert.strictEqual(typeof apiConfig, 'object');
assert.strictEqual(typeof pathPrefix, 'string');
assert.strictEqual(typeof callback, 'function');
shell.exec('remove', '/bin/rm', [ '-rf', pathPrefix ], { }, function (error) {
shell.exec('removeDir', '/bin/rm', [ '-rf', pathPrefix ], { }, function (error) {
if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message));
safe.fs.rmdirSync(path.dirname(pathPrefix)); // try to cleanup empty directories
src/storage/interface.js
+12 -1
View File
@@ -13,6 +13,7 @@ exports = module.exports = {
copy: copy,
remove: remove,
removeDir: removeDir,
backupDone: backupDone,
@@ -61,7 +62,17 @@ function copy(apiConfig, oldFilePath, newFilePath, callback) {
callback(new Error('not implemented'));
}
function remove(apiConfig, pathPrefix, callback) {
function remove(apiConfig, filename, callback) {
assert.strictEqual(typeof apiConfig, 'object');
assert.strictEqual(typeof filename, 'string');
assert.strictEqual(typeof callback, 'function');
// Result: none
callback(new Error('not implemented'));
}
function removeDir(apiConfig, pathPrefix, callback) {
assert.strictEqual(typeof apiConfig, 'object');
assert.strictEqual(typeof pathPrefix, 'string');
assert.strictEqual(typeof callback, 'function');
src/storage/noop.js
+13 -2
View File
@@ -7,6 +7,7 @@ exports = module.exports = {
copy: copy,
remove: remove,
removeDir: removeDir,
backupDone: backupDone,
@@ -59,12 +60,22 @@ function copy(apiConfig, oldFilePath, newFilePath, callback) {
callback();
}
function remove(apiConfig, pathPrefix, callback) {
function remove(apiConfig, filename, callback) {
assert.strictEqual(typeof apiConfig, 'object');
assert.strictEqual(typeof filename, 'string');
assert.strictEqual(typeof callback, 'function');
debug('remove: %s', filename);
callback();
}
function removeDir(apiConfig, pathPrefix, callback) {
assert.strictEqual(typeof apiConfig, 'object');
assert.strictEqual(typeof pathPrefix, 'string');
assert.strictEqual(typeof callback, 'function');
debug('remove: %s', pathPrefix);
debug('removeDir: %s', pathPrefix);
callback();
}
src/storage/s3.js
+31 -5
View File
@@ -7,6 +7,7 @@ exports = module.exports = {
copy: copy,
remove: remove,
removeDir: removeDir,
backupDone: backupDone,
@@ -253,7 +254,32 @@ function copy(apiConfig, oldFilePath, newFilePath, callback) {
});
}
function remove(apiConfig, pathPrefix, callback) {
function remove(apiConfig, filename, callback) {
assert.strictEqual(typeof apiConfig, 'object');
assert.strictEqual(typeof filename, 'string');
assert.strictEqual(typeof callback, 'function');
getBackupCredentials(apiConfig, function (error, credentials) {
if (error) return callback(error);
var s3 = new AWS.S3(credentials);
var deleteParams = {
Bucket: apiConfig.bucket,
Delete: {
Objects: [{ Key: filename }]
}
};
s3.deleteObjects(deleteParams, function (error) {
if (error) debug('remove: Unable to remove %s. Not fatal.', deleteParams.Key, error);
callback(null);
});
});
}
function removeDir(apiConfig, pathPrefix, callback) {
assert.strictEqual(typeof apiConfig, 'object');
assert.strictEqual(typeof pathPrefix, 'string');
assert.strictEqual(typeof callback, 'function');
@@ -270,7 +296,7 @@ function remove(apiConfig, pathPrefix, callback) {
async.forever(function listAndDelete(iteratorCallback) {
s3.listObjectsV2(listParams, function (error, listData) {
if (error) {
debug('remove: Failed to list %s. Not fatal.', error);
debug('removeDir: Failed to list %s. Not fatal.', error);
return iteratorCallback(error);
}
@@ -283,10 +309,10 @@ function remove(apiConfig, pathPrefix, callback) {
s3.deleteObjects(deleteParams, function (error, deleteData) {
if (error) {
debug('remove: Unable to remove %s. Not fatal.', deleteParams.Key, error);
debug('removeDir: Unable to remove %s. Not fatal.', deleteParams.Key, error);
return iteratorCallback(error);
}
debug(': Deleted: %j Errors: %j', deleteData.Deleted, deleteData.Errors);
debug('removeDir: Deleted: %j Errors: %j', deleteData.Deleted, deleteData.Errors);
listParams.Marker = listData.Contents[listData.Contents.length - 1].Key; // NextMarker is returned only with delimiter
@@ -306,7 +332,7 @@ function testConfig(apiConfig, callback) {
assert.strictEqual(typeof callback, 'function');
if (apiConfig.provider === 'caas') {
if (typeof apiConfig.token !== 'string') return callback(new BackupsError(BackupsError.BAD_FIELD, 'token must be a string'));
if (typeof apiConfig.token !== 'string') return callback(new BackupsError(BackupsError.BAD_FIELD, 'token must be a string'));
} else {
if (typeof apiConfig.accessKeyId !== 'string') return callback(new BackupsError(BackupsError.BAD_FIELD, 'accessKeyId must be a string'));
if (typeof apiConfig.secretAccessKey !== 'string') return callback(new BackupsError(BackupsError.BAD_FIELD, 'secretAccessKey must be a string'));
src/syncer.js
+2 -1
View File
@@ -40,7 +40,7 @@ function sync(dir, taskProcessor, concurrency, callback) {
var cacheFile = path.join(paths.BACKUP_INFO_DIR, path.basename(dir) + '.sync.cache'),
newCacheFile = path.join(paths.BACKUP_INFO_DIR, path.basename(dir) + '.sync.cache.new');
if (!safe.fs.existsSync(cacheFile)) { // if cache is missing, start out empty
if (!safe.fs.existsSync(cacheFile)) { // if cache is missing, start out empty. TODO: do a remote listDir and rebuild
delQueue.push({ operation: 'removedir', path: '' });
}
@@ -50,6 +50,7 @@ function sync(dir, taskProcessor, concurrency, callback) {
if (newCacheFd === -1) return callback(new Error('Error opening new cache file: ' + safe.error.message));
function advanceCache(entryPath) {
// TODO: detect and issue removedir
for (; curCacheIndex !== cache.length && (entryPath === '' || cache[curCacheIndex].path < entryPath); ++curCacheIndex) {
delQueue.push({ operation: 'remove', path: cache[curCacheIndex].path });
}
src/test/backups-test.js
+12 -7
View File
@@ -7,7 +7,6 @@
'use strict';
var async = require('async'),
appdb = require('../appdb.js'),
backupdb = require('../backupdb.js'),
backups = require('../backups.js'),
database = require('../database'),
@@ -46,7 +45,8 @@ describe('backups', function () {
version: '1.0.0',
type: backupdb.BACKUP_TYPE_BOX,
dependsOn: [ 'backup-app-00', 'backup-app-01' ],
restoreConfig: null
restoreConfig: null,
format: 'tgz'
};
var BACKUP_0_APP_0 = {
@@ -54,7 +54,8 @@ describe('backups', function () {
version: '1.0.0',
type: backupdb.BACKUP_TYPE_APP,
dependsOn: [],
restoreConfig: null
restoreConfig: null,
format: 'tgz'
};
var BACKUP_0_APP_1 = {
@@ -62,7 +63,8 @@ describe('backups', function () {
version: '1.0.0',
type: backupdb.BACKUP_TYPE_APP,
dependsOn: [],
restoreConfig: null
restoreConfig: null,
format: 'tgz'
};
var BACKUP_1 = {
@@ -70,7 +72,8 @@ describe('backups', function () {
version: '1.0.0',
type: backupdb.BACKUP_TYPE_BOX,
dependsOn: [ 'backup-app-10', 'backup-app-11' ],
restoreConfig: null
restoreConfig: null,
format: 'tgz'
};
var BACKUP_1_APP_0 = {
@@ -78,7 +81,8 @@ describe('backups', function () {
version: '1.0.0',
type: backupdb.BACKUP_TYPE_APP,
dependsOn: [],
restoreConfig: null
restoreConfig: null,
format: 'tgz'
};
var BACKUP_1_APP_1 = {
@@ -86,7 +90,8 @@ describe('backups', function () {
version: '1.0.0',
type: backupdb.BACKUP_TYPE_APP,
dependsOn: [],
restoreConfig: null
restoreConfig: null,
format: 'tgz'
};
it('succeeds without backups', function (done) {
src/test/database-test.js
+4 -2
View File
@@ -1025,7 +1025,8 @@ describe('database', function () {
version: '1.0.0',
type: backupdb.BACKUP_TYPE_BOX,
dependsOn: [ 'dep1' ],
restoreConfig: null
restoreConfig: null,
format: 'tgz'
};
backupdb.add(backup, function (error) {
@@ -1090,7 +1091,8 @@ describe('database', function () {
version: '1.0.0',
type: backupdb.BACKUP_TYPE_APP,
dependsOn: [ ],
restoreConfig: { manifest: { foo: 'bar' } }
restoreConfig: { manifest: { foo: 'bar' } },
format: 'tgz'
};
backupdb.add(backup, function (error) {
src/test/storage-test.js
+2 -2
View File
@@ -115,8 +115,8 @@ describe('Storage', function () {
gSourceFolder = path.join(__dirname, 'storage');
gDestinationFolder = path.join(gTmpFolder, 'destination/');
gBackupId_1 = backups._getBackupFilePath(gBackupConfig, 'someprefix/one');
gBackupId_2 = backups._getBackupFilePath(gBackupConfig, 'someprefix/two');
gBackupId_1 = backups._getBackupFilePath(gBackupConfig, 'someprefix/one', gBackupConfig.format);
gBackupId_2 = backups._getBackupFilePath(gBackupConfig, 'someprefix/two', gBackupConfig.format);
done();
});