diff --git a/dashboard/src/js/client.js b/dashboard/src/js/client.js index 21e7c6739..0d3048da0 100644 --- a/dashboard/src/js/client.js +++ b/dashboard/src/js/client.js @@ -1011,6 +1011,24 @@ angular.module('Application').service('Client', ['$http', '$interval', '$timeout }); }; + Client.prototype.setBackupPolicy = function (backupPolicy, callback) { + post('/api/v1/settings/backup_policy', backupPolicy, null, function (error, data, status) { + if (error) return callback(error); + if (status !== 200) return callback(new ClientError(status, data)); + + callback(null); + }); + }; + + Client.prototype.getBackupPolicy = function (callback) { + get('/api/v1/settings/backup_policy', null, function (error, data, status) { + if (error) return callback(error); + if (status !== 200) return callback(new ClientError(status, data)); + + callback(null, data.policy); + }); + }; + Client.prototype.getBackupMountStatus = function (callback) { get('/api/v1/backups/mount_status', null, function (error, data, status) { if (error) return callback(error); diff --git a/dashboard/src/views/backups.html b/dashboard/src/views/backups.html index f91d2a103..202284921 100644 --- a/dashboard/src/views/backups.html +++ b/dashboard/src/views/backups.html @@ -108,40 +108,40 @@ - diff --git a/dashboard/src/views/backups.js b/dashboard/src/views/backups.js index 4411be92b..42d6e74d6 100644 --- a/dashboard/src/views/backups.js +++ b/dashboard/src/views/backups.js @@ -35,7 +35,7 @@ angular.module('Application').controller('BackupsController', ['$scope', '$locat { name: 'No-op (Only for testing)', value: 'noop' } ]); - $scope.retentionPolicies = [ + $scope.backupRetentions = [ { name: '2 days', value: { keepWithinSecs: 2 * 24 * 60 * 60 }}, { name: '1 week', value: { keepWithinSecs: 7 * 24 * 60 * 60 }}, // default { name: '1 month', value: { keepWithinSecs: 30 * 24 * 60 * 60 }}, @@ -85,8 +85,8 @@ angular.module('Application').controller('BackupsController', ['$scope', '$locat return prettyDay + ' at ' + prettyHour; }; - $scope.prettyBackupRetentionPolicy = function (retentionPolicy) { - var tmp = $scope.retentionPolicies.find(function (p) { return angular.equals(p.value, retentionPolicy); }); + $scope.prettyBackupRetention = function (retention) { + var tmp = $scope.backupRetentions.find(function (p) { return angular.equals(p.value, retention); }); return tmp ? tmp.name : ''; }; @@ -347,68 +347,76 @@ angular.module('Application').controller('BackupsController', ['$scope', '$locat } }; - $scope.configureScheduleAndRetention = { + $scope.backupPolicy = { busy: false, error: {}, - retentionPolicy: $scope.retentionPolicies[0], + currentPolicy: null, + + retention: null, days: [], hours: [], + init: function () { + Client.getBackupPolicy(function (error, policy) { + if (error) Client.error(error); + $scope.backupPolicy.currentPolicy = policy; + }); + }, + show: function () { - $scope.configureScheduleAndRetention.error = {}; - $scope.configureScheduleAndRetention.busy = false; + $scope.backupPolicy.error = {}; + $scope.backupPolicy.busy = false; - var selectedPolicy = $scope.retentionPolicies.find(function (x) { return angular.equals(x.value, $scope.backupConfig.retentionPolicy); }); - if (!selectedPolicy) selectedPolicy = $scope.retentionPolicies[0]; + var selectedRetention = $scope.backupRetentions.find(function (x) { return angular.equals(x.value, $scope.backupPolicy.currentPolicy.retention); }); + if (!selectedRetention) selectedRetention = $scope.backupRetentions[0]; - $scope.configureScheduleAndRetention.retentionPolicy = selectedPolicy.value; + $scope.backupPolicy.retention = selectedRetention.value; - var tmp = $scope.backupConfig.schedulePattern.split(' '); + var tmp = $scope.backupPolicy.currentPolicy.schedule.split(' '); var hours = tmp[2].split(','), days = tmp[5].split(','); if (days[0] === '*') { - $scope.configureScheduleAndRetention.days = angular.copy($scope.cronDays, []); + $scope.backupPolicy.days = angular.copy($scope.cronDays, []); } else { - $scope.configureScheduleAndRetention.days = days.map(function (day) { return $scope.cronDays[parseInt(day, 10)]; }); + $scope.backupPolicy.days = days.map(function (day) { return $scope.cronDays[parseInt(day, 10)]; }); } - $scope.configureScheduleAndRetention.hours = hours.map(function (hour) { return $scope.cronHours[parseInt(hour, 10)]; }); + $scope.backupPolicy.hours = hours.map(function (hour) { return $scope.cronHours[parseInt(hour, 10)]; }); - $('#configureScheduleAndRetentionModal').modal('show'); + $('#backupPolicyModal').modal('show'); }, valid: function () { - return $scope.configureScheduleAndRetention.days.length && $scope.configureScheduleAndRetention.hours.length; + return $scope.backupPolicy.days.length && $scope.backupPolicy.hours.length; }, submit: function () { - if (!$scope.configureScheduleAndRetention.days.length) return; - if (!$scope.configureScheduleAndRetention.hours.length) return; + if (!$scope.backupPolicy.days.length) return; + if (!$scope.backupPolicy.hours.length) return; - $scope.configureScheduleAndRetention.error = {}; - $scope.configureScheduleAndRetention.busy = true; - - // start with the full backupConfig since the api requires all fields - var backupConfig = $scope.backupConfig; - backupConfig.retentionPolicy = $scope.configureScheduleAndRetention.retentionPolicy; + $scope.backupPolicy.error = {}; + $scope.backupPolicy.busy = true; var daysPattern; - if ($scope.configureScheduleAndRetention.days.length === 7) daysPattern = '*'; - else daysPattern = $scope.configureScheduleAndRetention.days.map(function (d) { return d.value; }); + if ($scope.backupPolicy.days.length === 7) daysPattern = '*'; + else daysPattern = $scope.backupPolicy.days.map(function (d) { return d.value; }); var hoursPattern; - if ($scope.configureScheduleAndRetention.hours.length === 24) hoursPattern = '*'; - else hoursPattern = $scope.configureScheduleAndRetention.hours.map(function (d) { return d.value; }); + if ($scope.backupPolicy.hours.length === 24) hoursPattern = '*'; + else hoursPattern = $scope.backupPolicy.hours.map(function (d) { return d.value; }); - backupConfig.schedulePattern ='00 00 ' + hoursPattern + ' * * ' + daysPattern; + var policy = { + retention: $scope.backupPolicy.retention, + schedule: '00 00 ' + hoursPattern + ' * * ' + daysPattern + }; - Client.setBackupConfig(backupConfig, function (error) { - $scope.configureScheduleAndRetention.busy = false; + Client.setBackupPolicy(policy, function (error) { + $scope.backupPolicy.busy = false; if (error) { if (error.statusCode === 424) { - $scope.configureScheduleAndRetention.error.generic = error.message; + $scope.backupPolicy.error.generic = error.message; } else if (error.statusCode === 400) { - $scope.configureScheduleAndRetention.error.generic = error.message; + $scope.backupPolicy.error.generic = error.message; } else { console.error('Unable to change schedule or retention.', error); } @@ -416,9 +424,9 @@ angular.module('Application').controller('BackupsController', ['$scope', '$locat return; } - $('#configureScheduleAndRetentionModal').modal('hide'); + $('#backupPolicyModal').modal('hide'); - getBackupConfig(); + $scope.backupPolicy.init(); }); } }; @@ -813,6 +821,7 @@ angular.module('Application').controller('BackupsController', ['$scope', '$locat // show backup status $scope.createBackup.init(); $scope.cleanupBackups.init(); + $scope.backupPolicy.init(); getBackupTasks(); getCleanupTasks(); diff --git a/migrations/20230712042655-settings-split-backup-config-policy.js b/migrations/20230712042655-settings-split-backup-config-policy.js new file mode 100644 index 000000000..cfc1e2151 --- /dev/null +++ b/migrations/20230712042655-settings-split-backup-config-policy.js @@ -0,0 +1,19 @@ +'use strict'; + +exports.up = async function(db) { + const result = await db.runSql('SELECT * FROM settings WHERE name=?', [ 'backup_config' ]); + if (!result.length) return; + + const backupConfig = JSON.parse(result[0].value); + const backupPolicy = { schedule: backupConfig.schedulePattern, retention: backupConfig.retentionPolicy }; + delete backupConfig.schedulePattern; + delete backupConfig.retentionPolicy; + + await db.runSql('START TRANSACTION'); + await db.runSql('UPDATE settings SET value=? WHERE name=?', [ JSON.stringify(backupConfig), 'backup_config']); + await db.runSql('UPDATE settings SET value=? WHERE name=?', [ JSON.stringify(backupPolicy), 'backup_policy']); + await db.runSql('COMMIT'); +}; + +exports.down = async function(/* db */) { +}; diff --git a/src/backupcleaner.js b/src/backupcleaner.js index 3f7d9758e..bbc09e19e 100644 --- a/src/backupcleaner.js +++ b/src/backupcleaner.js @@ -5,7 +5,7 @@ const BoxError = require('./boxerror.js'); exports = module.exports = { run, - _applyBackupRetentionPolicy: applyBackupRetentionPolicy + _applyBackupRetention: applyBackupRetention }; const apps = require('./apps.js'), @@ -21,9 +21,9 @@ const apps = require('./apps.js'), settings = require('./settings.js'), storage = require('./storage.js'); -function applyBackupRetentionPolicy(allBackups, policy, referencedBackupIds) { +function applyBackupRetention(allBackups, retention, referencedBackupIds) { assert(Array.isArray(allBackups)); - assert.strictEqual(typeof policy, 'object'); + assert.strictEqual(typeof retention, 'object'); assert(Array.isArray(referencedBackupIds)); const now = new Date(); @@ -38,7 +38,7 @@ function applyBackupRetentionPolicy(allBackups, policy, referencedBackupIds) { backup.keepReason = 'referenced'; } else if ((backup.preserveSecs === -1) || ((now - backup.creationTime) < (backup.preserveSecs * 1000))) { backup.keepReason = 'preserveSecs'; - } else if ((now - backup.creationTime < policy.keepWithinSecs * 1000) || policy.keepWithinSecs < 0) { + } else if ((now - backup.creationTime < retention.keepWithinSecs * 1000) || retention.keepWithinSecs < 0) { backup.keepReason = 'keepWithinSecs'; } } @@ -51,9 +51,9 @@ function applyBackupRetentionPolicy(allBackups, policy, referencedBackupIds) { }; for (const format of [ 'keepDaily', 'keepWeekly', 'keepMonthly', 'keepYearly' ]) { - if (!(format in policy)) continue; + if (!(format in retention)) continue; - const n = policy[format]; // we want to keep "n" backups of format + const n = retention[format]; // we want to keep "n" backups of format if (!n) continue; // disabled rule let lastPeriod = null, keptSoFar = 0; @@ -69,7 +69,7 @@ function applyBackupRetentionPolicy(allBackups, policy, referencedBackupIds) { } } - if (policy.keepLatest) { + if (retention.keepLatest) { let latestNormalBackup = allBackups.find(b => b.state === backups.BACKUP_STATE_NORMAL); if (latestNormalBackup && !latestNormalBackup.keepReason) latestNormalBackup.keepReason = 'latest'; } @@ -109,8 +109,9 @@ async function removeBackup(backupConfig, backup, progressCallback) { else debug(`removeBackup: removed ${backup.remotePath}`); } -async function cleanupAppBackups(backupConfig, referencedBackupIds, progressCallback) { +async function cleanupAppBackups(backupConfig, retention, referencedBackupIds, progressCallback) { assert.strictEqual(typeof backupConfig, 'object'); + assert.strictEqual(typeof retention, 'object'); assert(Array.isArray(referencedBackupIds)); assert.strictEqual(typeof progressCallback, 'function'); @@ -131,9 +132,9 @@ async function cleanupAppBackups(backupConfig, referencedBackupIds, progressCall // apply backup policy per app. keep latest backup only for existing apps let appBackupsToRemove = []; for (const appId of Object.keys(appBackupsById)) { - const policy = Object.assign({ keepLatest: allAppIds.includes(appId) }, backupConfig.retentionPolicy); - debug(`cleanupAppBackups: applying policy for appId ${appId} policy: ${JSON.stringify(policy)}`); - applyBackupRetentionPolicy(appBackupsById[appId], policy, referencedBackupIds); + const appRetention = Object.assign({ keepLatest: allAppIds.includes(appId) }, retention); + debug(`cleanupAppBackups: applying retention for appId ${appId} retention: ${JSON.stringify(appRetention)}`); + applyBackupRetention(appBackupsById[appId], appRetention, referencedBackupIds); appBackupsToRemove = appBackupsToRemove.concat(appBackupsById[appId].filter(b => !b.keepReason)); } @@ -148,8 +149,9 @@ async function cleanupAppBackups(backupConfig, referencedBackupIds, progressCall return removedAppBackupPaths; } -async function cleanupMailBackups(backupConfig, referencedBackupIds, progressCallback) { +async function cleanupMailBackups(backupConfig, retention, referencedBackupIds, progressCallback) { assert.strictEqual(typeof backupConfig, 'object'); + assert.strictEqual(typeof retention, 'object'); assert(Array.isArray(referencedBackupIds)); assert.strictEqual(typeof progressCallback, 'function'); @@ -157,7 +159,7 @@ async function cleanupMailBackups(backupConfig, referencedBackupIds, progressCal const mailBackups = await backups.getByTypePaged(backups.BACKUP_TYPE_MAIL, 1, 1000); - applyBackupRetentionPolicy(mailBackups, Object.assign({ keepLatest: true }, backupConfig.retentionPolicy), referencedBackupIds); + applyBackupRetention(mailBackups, Object.assign({ keepLatest: true }, retention), referencedBackupIds); for (const mailBackup of mailBackups) { if (mailBackup.keepReason) continue; @@ -171,15 +173,16 @@ async function cleanupMailBackups(backupConfig, referencedBackupIds, progressCal return removedMailBackupPaths; } -async function cleanupBoxBackups(backupConfig, progressCallback) { +async function cleanupBoxBackups(backupConfig, retention, progressCallback) { assert.strictEqual(typeof backupConfig, 'object'); + assert.strictEqual(typeof retention, 'object'); assert.strictEqual(typeof progressCallback, 'function'); let referencedBackupIds = [], removedBoxBackupPaths = []; const boxBackups = await backups.getByTypePaged(backups.BACKUP_TYPE_BOX, 1, 1000); - applyBackupRetentionPolicy(boxBackups, Object.assign({ keepLatest: true }, backupConfig.retentionPolicy), [] /* references */); + applyBackupRetention(boxBackups, Object.assign({ keepLatest: true }, retention), [] /* references */); for (const boxBackup of boxBackups) { if (boxBackup.keepReason) { @@ -271,24 +274,25 @@ async function run(progressCallback) { assert.strictEqual(typeof progressCallback, 'function'); const backupConfig = await settings.getBackupConfig(); + const { retention } = await settings.getBackupPolicy(); const status = await storage.api(backupConfig.provider).getBackupProviderStatus(backupConfig); debug(`clean: mount point status is ${JSON.stringify(status)}`); if (status.state !== 'active') throw new BoxError(BoxError.MOUNT_ERROR, `Backup endpoint is not mounted: ${status.message}`); - if (backupConfig.retentionPolicy.keepWithinSecs < 0) { + if (retention.keepWithinSecs < 0) { debug('cleanup: keeping all backups'); return {}; } await progressCallback({ percent: 10, message: 'Cleaning box backups' }); - const { removedBoxBackupPaths, referencedBackupIds } = await cleanupBoxBackups(backupConfig, progressCallback); // references is app or mail backup ids + const { removedBoxBackupPaths, referencedBackupIds } = await cleanupBoxBackups(backupConfig, retention, progressCallback); // references is app or mail backup ids await progressCallback({ percent: 20, message: 'Cleaning mail backups' }); - const removedMailBackupPaths = await cleanupMailBackups(backupConfig, referencedBackupIds, progressCallback); + const removedMailBackupPaths = await cleanupMailBackups(backupConfig, retention, referencedBackupIds, progressCallback); await progressCallback({ percent: 40, message: 'Cleaning app backups' }); - const removedAppBackupPaths = await cleanupAppBackups(backupConfig, referencedBackupIds, progressCallback); + const removedAppBackupPaths = await cleanupAppBackups(backupConfig, retention, referencedBackupIds, progressCallback); await progressCallback({ percent: 70, message: 'Checking storage backend and removing stale entries in database' }); const missingBackupPaths = await cleanupMissingBackups(backupConfig, progressCallback); diff --git a/src/backups.js b/src/backups.js index 56e2599f1..5affa8848 100644 --- a/src/backups.js +++ b/src/backups.js @@ -23,6 +23,7 @@ exports = module.exports = { getSnapshotInfo, setSnapshotInfo, + validatePolicy, testConfig, testProviderConfig, @@ -177,6 +178,22 @@ function validateLabel(label) { return null; } +async function validatePolicy(policy) { + assert.strictEqual(typeof policy, 'object'); + + const job = safe.safeCall(function () { return new CronJob(policy.schedule); }); + if (!job) return new BoxError(BoxError.BAD_FIELD, 'Invalid schedule pattern'); + + const retention = policy.retention; + if (!retention) return new BoxError(BoxError.BAD_FIELD, 'retention is required'); + if (!['keepWithinSecs','keepDaily','keepWeekly','keepMonthly','keepYearly'].find(k => !!retention[k])) return new BoxError(BoxError.BAD_FIELD, 'retention properties missing'); + if ('keepWithinSecs' in retention && typeof retention.keepWithinSecs !== 'number') return new BoxError(BoxError.BAD_FIELD, 'retention.keepWithinSecs must be a number'); + if ('keepDaily' in retention && typeof retention.keepDaily !== 'number') return new BoxError(BoxError.BAD_FIELD, 'retention.keepDaily must be a number'); + if ('keepWeekly' in retention && typeof retention.keepWeekly !== 'number') return new BoxError(BoxError.BAD_FIELD, 'retention.keepWeekly must be a number'); + if ('keepMonthly' in retention && typeof retention.keepMonthly !== 'number') return new BoxError(BoxError.BAD_FIELD, 'retention.keepMonthly must be a number'); + if ('keepYearly' in retention && typeof retention.keepYearly !== 'number') return new BoxError(BoxError.BAD_FIELD, 'retention.keepYearly must be a number'); +} + // this is called by REST API async function update(id, data) { assert.strictEqual(typeof id, 'string'); @@ -323,23 +340,11 @@ async function testConfig(backupConfig) { if (backupConfig.format !== 'tgz' && backupConfig.format !== 'rsync') return new BoxError(BoxError.BAD_FIELD, 'unknown format'); - const job = safe.safeCall(function () { return new CronJob(backupConfig.schedulePattern); }); - if (!job) return new BoxError(BoxError.BAD_FIELD, 'Invalid schedule pattern'); - if ('password' in backupConfig) { if (typeof backupConfig.password !== 'string') return new BoxError(BoxError.BAD_FIELD, 'password must be a string'); if (backupConfig.password.length < 8) return new BoxError(BoxError.BAD_FIELD, 'password must be atleast 8 characters'); } - const policy = backupConfig.retentionPolicy; - if (!policy) return new BoxError(BoxError.BAD_FIELD, 'retentionPolicy is required'); - if (!['keepWithinSecs','keepDaily','keepWeekly','keepMonthly','keepYearly'].find(k => !!policy[k])) return new BoxError(BoxError.BAD_FIELD, 'properties missing'); - if ('keepWithinSecs' in policy && typeof policy.keepWithinSecs !== 'number') return new BoxError(BoxError.BAD_FIELD, 'keepWithinSecs must be a number'); - if ('keepDaily' in policy && typeof policy.keepDaily !== 'number') return new BoxError(BoxError.BAD_FIELD, 'keepDaily must be a number'); - if ('keepWeekly' in policy && typeof policy.keepWeekly !== 'number') return new BoxError(BoxError.BAD_FIELD, 'keepWeekly must be a number'); - if ('keepMonthly' in policy && typeof policy.keepMonthly !== 'number') return new BoxError(BoxError.BAD_FIELD, 'keepMonthly must be a number'); - if ('keepYearly' in policy && typeof policy.keepYearly !== 'number') return new BoxError(BoxError.BAD_FIELD, 'keepYearly must be a number'); - await storage.api(backupConfig.provider).testConfig(backupConfig); } diff --git a/src/cron.js b/src/cron.js index 8f98bf7dd..88d77715c 100644 --- a/src/cron.js +++ b/src/cron.js @@ -161,7 +161,7 @@ async function startJobs() { const allSettings = await settings.list(); const tz = allSettings[settings.TIME_ZONE_KEY]; - backupConfigChanged(allSettings[settings.BACKUP_CONFIG_KEY], tz); + backupPolicyChanged(allSettings[settings.BACKUP_POLICY_KEY], tz); autoupdatePatternChanged(allSettings[settings.AUTOUPDATE_PATTERN_KEY], tz); dynamicDnsChanged(allSettings[settings.DYNAMIC_DNS_KEY]); } @@ -185,16 +185,16 @@ async function handleSettingsChanged(key, value) { } } -function backupConfigChanged(value, tz) { +function backupPolicyChanged(value, tz) { assert.strictEqual(typeof value, 'object'); assert.strictEqual(typeof tz, 'string'); - debug(`backupConfigChanged: schedule ${value.schedulePattern} (${tz})`); + debug(`backupPolicyChanged: schedule ${value.schedule} (${tz})`); if (gJobs.backup) gJobs.backup.stop(); gJobs.backup = new CronJob({ - cronTime: value.schedulePattern, + cronTime: value.schedule, onTick: async () => await safe(backups.startBackupTask(AuditSource.CRON), { debug }), start: true, timeZone: tz diff --git a/src/routes/settings.js b/src/routes/settings.js index 71d94a8a1..d6fb1f53c 100644 --- a/src/routes/settings.js +++ b/src/routes/settings.js @@ -72,7 +72,6 @@ async function setBackupConfig(req, res, next) { assert.strictEqual(typeof req.body, 'object'); if (typeof req.body.provider !== 'string') return next(new HttpError(400, 'provider is required')); - if (typeof req.body.schedulePattern !== 'string') return next(new HttpError(400, 'schedulePattern is required')); if ('password' in req.body && typeof req.body.password !== 'string') return next(new HttpError(400, 'password must be a string')); if ('encryptedFilenames' in req.body && typeof req.body.encryptedFilenames !== 'boolean') return next(new HttpError(400, 'encryptedFilenames must be a boolean')); @@ -101,8 +100,6 @@ async function setBackupConfig(req, res, next) { if (typeof req.body.format !== 'string') return next(new HttpError(400, 'format must be a string')); if ('acceptSelfSignedCerts' in req.body && typeof req.body.acceptSelfSignedCerts !== 'boolean') return next(new HttpError(400, 'format must be a boolean')); - if (!req.body.retentionPolicy || typeof req.body.retentionPolicy !== 'object') return next(new HttpError(400, 'retentionPolicy is required')); - if ('mountOptions' in req.body && typeof req.body.mountOptions !== 'object') return next(new HttpError(400, 'mountOptions must be a object')); // testing the backup using put/del takes a bit of time at times @@ -177,6 +174,25 @@ async function setDynamicDnsConfig(req, res, next) { next(new HttpSuccess(200, {})); } +async function getBackupPolicy(req, res, next) { + const [error, policy] = await safe(settings.getBackupPolicy()); + if (error) return next(BoxError.toHttpError(error)); + + next(new HttpSuccess(200, { policy })); +} + +async function setBackupPolicy(req, res, next) { + assert.strictEqual(typeof req.body, 'object'); + + if (typeof req.body.schedule !== 'string') return next(new HttpError(400, 'schedule is required')); + if (!req.body.retention || typeof req.body.retention !== 'object') return next(new HttpError(400, 'retention is required')); + + const [error] = await safe(settings.setBackupPolicy(req.body)); + if (error) return next(BoxError.toHttpError(error)); + + next(new HttpSuccess(200, {})); +} + async function getIPv6Config(req, res, next) { const [error, ipv6Config] = await safe(settings.getIPv6Config()); if (error) return next(BoxError.toHttpError(error)); @@ -296,6 +312,7 @@ function get(req, res, next) { assert.strictEqual(typeof req.params.setting, 'string'); switch (req.params.setting) { + case settings.BACKUP_POLICY_KEY: return getBackupPolicy(req, res, next); case settings.DYNAMIC_DNS_KEY: return getDynamicDnsConfig(req, res, next); case settings.IPV6_CONFIG_KEY: return getIPv6Config(req, res, next); case settings.BACKUP_CONFIG_KEY: return getBackupConfig(req, res, next); @@ -320,6 +337,7 @@ function set(req, res, next) { assert.strictEqual(typeof req.body, 'object'); switch (req.params.setting) { + case settings.BACKUP_POLICY_KEY: return setBackupPolicy(req, res, next); case settings.DYNAMIC_DNS_KEY: return setDynamicDnsConfig(req, res, next); case settings.IPV6_CONFIG_KEY: return setIPv6Config(req, res, next); case settings.EXTERNAL_LDAP_KEY: return setExternalLdapConfig(req, res, next); diff --git a/src/routes/test/backups-test.js b/src/routes/test/backups-test.js index 34e6a5d1d..ac8e6805f 100644 --- a/src/routes/test/backups-test.js +++ b/src/routes/test/backups-test.js @@ -23,8 +23,6 @@ describe('Backups API', function () { backupFolder: '/tmp/backups', format: 'tgz', encryption: null, - retentionPolicy: { keepWithinSecs: 2 * 24 * 60 * 60 }, // 2 days - schedulePattern: '00 00 23 * * *' // every day at 11pm }); }); diff --git a/src/routes/test/settings-test.js b/src/routes/test/settings-test.js index df4abd183..e860477e9 100644 --- a/src/routes/test/settings-test.js +++ b/src/routes/test/settings-test.js @@ -169,6 +169,97 @@ describe('Settings API', function () { }); }); + describe('backup_policy', function () { + const defaultPolicy = { + retention: { keepWithinSecs: 2 * 24 * 60 * 60 }, // 2 days + schedule: '00 00 23 * * *' // every day at 11pm + }; + + it('cannot set backup_policy without schedule', async function () { + const tmp = Object.assign({} , defaultPolicy); + delete tmp.schedule; + + const response = await superagent.post(`${serverUrl}/api/v1/settings/backup_policy`) + .query({ access_token: owner.token }) + .send(tmp) + .ok(() => true); + + expect(response.statusCode).to.equal(400); + }); + + it('cannot set backup_policy with invalid schedule', async function () { + const tmp = Object.assign({} , defaultPolicy); + tmp.schedule = 'not a pattern'; + + const response = await superagent.post(`${serverUrl}/api/v1/settings/backup_policy`) + .query({ access_token: owner.token }) + .send(tmp) + .ok(() => true); + + expect(response.statusCode).to.equal(400); + }); + + it('cannot set backup_policy without retention', async function () { + const tmp = Object.assign({} , defaultPolicy); + delete tmp.retention; + + const response = await superagent.post(`${serverUrl}/api/v1/settings/backup_policy`) + .query({ access_token: owner.token }) + .send(tmp) + .ok(() => true); + + expect(response.statusCode).to.equal(400); + }); + + it('cannot set backup_policy with invalid retention', async function () { + const tmp = Object.assign({} , defaultPolicy); + tmp.retention = 'not an object'; + + const response = await superagent.post(`${serverUrl}/api/v1/settings/backup_policy`) + .query({ access_token: owner.token }) + .send(tmp) + .ok(() => true); + + expect(response.statusCode).to.equal(400); + }); + + it('cannot set backup_policy with empty retention', async function () { + const tmp = Object.assign({} , defaultPolicy); + tmp.retention = {}; + + const response = await superagent.post(`${serverUrl}/api/v1/settings/backup_policy`) + .query({ access_token: owner.token }) + .send(tmp) + .ok(() => true); + + expect(response.statusCode).to.equal(400); + }); + + it('cannot set backup_policy with retention missing properties', async function () { + const tmp = Object.assign({} , defaultPolicy); + tmp.retention = { foo: 'bar' }; + + const response = await superagent.post(`${serverUrl}/api/v1/settings/backup_policy`) + .query({ access_token: owner.token }) + .send(tmp) + .ok(() => true); + + expect(response.statusCode).to.equal(400); + }); + + it('cannot set backup_policy with retention with invalid keepWithinSecs', async function () { + const tmp = Object.assign({} , defaultPolicy); + tmp.retention = { keepWithinSecs: 'not a number' }; + + const response = await superagent.post(`${serverUrl}/api/v1/settings/backup_policy`) + .query({ access_token: owner.token }) + .send(tmp) + .ok(() => true); + + expect(response.statusCode).to.equal(400); + }); + }); + describe('backup_config', function () { // keep in sync with defaults in settings.js let defaultConfig = { @@ -176,8 +267,6 @@ describe('Settings API', function () { backupFolder: '/var/backups', format: 'tgz', encryption: null, - retentionPolicy: { keepWithinSecs: 2 * 24 * 60 * 60 }, // 2 days - schedulePattern: '00 00 23 * * *' // every day at 11pm }; it('can get backup_config (default)', async function () { @@ -212,30 +301,6 @@ describe('Settings API', function () { expect(response.statusCode).to.equal(400); }); - it('cannot set backup_config without schedulePattern', async function () { - let tmp = JSON.parse(JSON.stringify(defaultConfig)); - delete tmp.schedulePattern; - - const response = await superagent.post(`${serverUrl}/api/v1/settings/backup_config`) - .query({ access_token: owner.token }) - .send(tmp) - .ok(() => true); - - expect(response.statusCode).to.equal(400); - }); - - it('cannot set backup_config with invalid schedulePattern', async function () { - let tmp = JSON.parse(JSON.stringify(defaultConfig)); - tmp.schedulePattern = 'not a pattern'; - - const response = await superagent.post(`${serverUrl}/api/v1/settings/backup_config`) - .query({ access_token: owner.token }) - .send(tmp) - .ok(() => true); - - expect(response.statusCode).to.equal(400); - }); - it('cannot set backup_config without format', async function () { let tmp = JSON.parse(JSON.stringify(defaultConfig)); delete tmp.format; @@ -260,66 +325,6 @@ describe('Settings API', function () { expect(response.statusCode).to.equal(400); }); - it('cannot set backup_config without retentionPolicy', async function () { - let tmp = JSON.parse(JSON.stringify(defaultConfig)); - delete tmp.retentionPolicy; - - const response = await superagent.post(`${serverUrl}/api/v1/settings/backup_config`) - .query({ access_token: owner.token }) - .send(tmp) - .ok(() => true); - - expect(response.statusCode).to.equal(400); - }); - - it('cannot set backup_config with invalid retentionPolicy', async function () { - let tmp = JSON.parse(JSON.stringify(defaultConfig)); - tmp.retentionPolicy = 'not an object'; - - const response = await superagent.post(`${serverUrl}/api/v1/settings/backup_config`) - .query({ access_token: owner.token }) - .send(tmp) - .ok(() => true); - - expect(response.statusCode).to.equal(400); - }); - - it('cannot set backup_config with empty retentionPolicy', async function () { - let tmp = JSON.parse(JSON.stringify(defaultConfig)); - tmp.retentionPolicy = {}; - - const response = await superagent.post(`${serverUrl}/api/v1/settings/backup_config`) - .query({ access_token: owner.token }) - .send(tmp) - .ok(() => true); - - expect(response.statusCode).to.equal(400); - }); - - it('cannot set backup_config with retentionPolicy missing properties', async function () { - let tmp = JSON.parse(JSON.stringify(defaultConfig)); - tmp.retentionPolicy = { foo: 'bar' }; - - const response = await superagent.post(`${serverUrl}/api/v1/settings/backup_config`) - .query({ access_token: owner.token }) - .send(tmp) - .ok(() => true); - - expect(response.statusCode).to.equal(400); - }); - - it('cannot set backup_config with retentionPolicy with invalid keepWithinSecs', async function () { - let tmp = JSON.parse(JSON.stringify(defaultConfig)); - tmp.retentionPolicy = { keepWithinSecs: 'not a number' }; - - const response = await superagent.post(`${serverUrl}/api/v1/settings/backup_config`) - .query({ access_token: owner.token }) - .send(tmp) - .ok(() => true); - - expect(response.statusCode).to.equal(400); - }); - it('cannot set backup_config with invalid password', async function () { let tmp = JSON.parse(JSON.stringify(defaultConfig)); tmp.password = 1234; diff --git a/src/settings.js b/src/settings.js index 5b02257ea..5649c1fe7 100644 --- a/src/settings.js +++ b/src/settings.js @@ -24,6 +24,9 @@ exports = module.exports = { getUnstableAppsConfig, setUnstableAppsConfig, + getBackupPolicy, + setBackupPolicy, + getBackupConfig, setBackupConfig, setBackupCredentials, @@ -101,6 +104,7 @@ exports = module.exports = { // json. if you add an entry here, be sure to fix list() BACKUP_CONFIG_KEY: 'backup_config', + BACKUP_POLICY_KEY: 'backup_policy', SERVICES_CONFIG_KEY: 'services_config', EXTERNAL_LDAP_KEY: 'external_ldap_config', DIRECTORY_SERVER_KEY: 'user_directory_config', @@ -187,8 +191,10 @@ const gDefaults = (function () { backupFolder: '/var/backups', format: 'tgz', encryption: null, - retentionPolicy: { keepWithinSecs: 2 * 24 * 60 * 60 }, // 2 days - schedulePattern: '00 00 23 * * *' // every day at 11pm + }; + result[exports.BACKUP_POLICY_KEY] = { + retention: { keepWithinSecs: 2 * 24 * 60 * 60 }, // 2 days + schedule: '00 00 23 * * *' // every day at 11pm }; result[exports.REVERSE_PROXY_CONFIG_KEY] = { ocsp: true @@ -407,6 +413,22 @@ async function setUnstableAppsConfig(enabled) { notifyChange(exports.UNSTABLE_APPS_KEY, enabled); } +async function getBackupPolicy() { + const result = await get(exports.BACKUP_POLICY_KEY); + if (result === null) return gDefaults[exports.BACKUP_POLICY_KEY]; + return JSON.parse(result); +} + +async function setBackupPolicy(policy) { + assert.strictEqual(typeof policy, 'object'); + + const error = await backups.validatePolicy(policy); + if (error) throw error; + + await set(exports.BACKUP_POLICY_KEY, JSON.stringify(policy)); + notifyChange(exports.BACKUP_POLICY_KEY, policy); +} + async function getBackupConfig() { const value = await get(exports.BACKUP_CONFIG_KEY); if (value === null) return gDefaults[exports.BACKUP_CONFIG_KEY]; @@ -469,7 +491,7 @@ async function setBackupCredentials(credentials) { const currentConfig = await getBackupConfig(); // preserve these fields - const extra = _.pick(currentConfig, 'retentionPolicy', 'schedulePattern', 'copyConcurrency', 'syncConcurrency', 'memoryLimit', 'downloadConcurrency', 'deleteConcurrency', 'uploadPartSize'); + const extra = _.pick(currentConfig, 'copyConcurrency', 'syncConcurrency', 'memoryLimit', 'downloadConcurrency', 'deleteConcurrency', 'uploadPartSize'); const backupConfig = Object.assign({}, credentials, extra); @@ -743,7 +765,8 @@ async function list() { result[exports.DEMO_KEY] = !!result[exports.DEMO_KEY]; // convert JSON objects - [exports.BACKUP_CONFIG_KEY, exports.IPV6_CONFIG_KEY, exports.PROFILE_CONFIG_KEY, exports.SERVICES_CONFIG_KEY, exports.EXTERNAL_LDAP_KEY, exports.REGISTRY_CONFIG_KEY, exports.SYSINFO_CONFIG_KEY, exports.REVERSE_PROXY_CONFIG_KEY ].forEach(function (key) { + [exports.BACKUP_POLICY_KEY, exports.BACKUP_CONFIG_KEY, exports.IPV6_CONFIG_KEY, exports.PROFILE_CONFIG_KEY, exports.SERVICES_CONFIG_KEY, + exports.EXTERNAL_LDAP_KEY, exports.REGISTRY_CONFIG_KEY, exports.SYSINFO_CONFIG_KEY, exports.REVERSE_PROXY_CONFIG_KEY ].forEach(function (key) { result[key] = typeof result[key] === 'object' ? result[key] : safe.JSON.parse(result[key]); }); diff --git a/src/test/backupcleaner-test.js b/src/test/backupcleaner-test.js index 2557f598a..8fe4371e6 100644 --- a/src/test/backupcleaner-test.js +++ b/src/test/backupcleaner-test.js @@ -35,28 +35,28 @@ describe('backup cleaner', function () { preserveSecs: 0 }; - describe('retention policy', function () { + describe('retention', function () { it('keeps latest', function () { const backup = Object.assign({}, backupTemplate, { creationTime: moment().subtract(5, 's').toDate(), state: backups.BACKUP_STATE_NORMAL }); - backupCleaner._applyBackupRetentionPolicy([backup], { keepWithinSecs: 1, keepLatest: true }, []); + backupCleaner._applyBackupRetention([backup], { keepWithinSecs: 1, keepLatest: true }, []); expect(backup.keepReason).to.be('latest'); }); it('does not keep latest', function () { let backup = { creationTime: moment().subtract(5, 's').toDate(), state: backups.BACKUP_STATE_NORMAL }; - backupCleaner._applyBackupRetentionPolicy([backup], { keepWithinSecs: 1, keepLatest: false }, []); + backupCleaner._applyBackupRetention([backup], { keepWithinSecs: 1, keepLatest: false }, []); expect(backup.keepReason).to.be(undefined); }); it('always keeps forever policy', function () { let backup = { creationTime: new Date() }; - backupCleaner._applyBackupRetentionPolicy([backup], { keepWithinSecs: -1, keepLatest: true }, []); + backupCleaner._applyBackupRetention([backup], { keepWithinSecs: -1, keepLatest: true }, []); expect(backup.keepReason).to.be('keepWithinSecs'); }); it('preserveSecs takes precedence', function () { let backup = { creationTime: new Date(), preserveSecs: 3000 }; - backupCleaner._applyBackupRetentionPolicy([backup], { keepWithinSecs: 1, keepLatest: true }, []); + backupCleaner._applyBackupRetention([backup], { keepWithinSecs: 1, keepLatest: true }, []); expect(backup.keepReason).to.be('preserveSecs'); }); @@ -68,7 +68,7 @@ describe('backup cleaner', function () { { id: '3', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(20, 'h').toDate() }, { id: '4', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(5, 'd').toDate() } ]; - backupCleaner._applyBackupRetentionPolicy(b, { keepDaily: 1, keepLatest: true }, []); + backupCleaner._applyBackupRetention(b, { keepDaily: 1, keepLatest: true }, []); expect(b[0].keepReason).to.be('keepDaily'); expect(b[1].keepReason).to.be(undefined); expect(b[2].keepReason).to.be(undefined); @@ -87,7 +87,7 @@ describe('backup cleaner', function () { { id: '5', state: backups.BACKUP_STATE_CREATING, creationTime: moment().subtract(50, 'h').toDate() }, { id: '6', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(5, 'd').toDate() } ]; - backupCleaner._applyBackupRetentionPolicy(b, { keepDaily: 2, keepWeekly: 1, keepLatest: false }, []); + backupCleaner._applyBackupRetention(b, { keepDaily: 2, keepWeekly: 1, keepLatest: false }, []); expect(b[0].keepReason).to.be('keepDaily'); // today expect(b[1].keepReason).to.be('keepWeekly'); // today expect(b[2].keepReason).to.be(undefined); @@ -110,7 +110,7 @@ describe('backup cleaner', function () { { id: '8', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(84, 'd').toDate() }, { id: '9', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(97, 'd').toDate() }, ]; - backupCleaner._applyBackupRetentionPolicy(b, { keepDaily: 2, keepMonthly: 3, keepYearly: 1, keepLatest: true }, []); + backupCleaner._applyBackupRetention(b, { keepDaily: 2, keepMonthly: 3, keepYearly: 1, keepLatest: true }, []); expect(b[0].keepReason).to.be('creating'); expect(b[1].discardReason).to.be('error'); // errored expect(b[2].keepReason).to.be('keepDaily'); @@ -214,9 +214,9 @@ describe('backup cleaner', function () { provider: 'filesystem', password: 'supersecret', backupFolder: '/tmp/someplace', - retentionPolicy: { keepWithinSecs: 1 }, format: 'tgz' })); + await settings.setBackupPolicy({ retention: { keepWithinSecs: 1 }, schedule: '00 00 23 * * *' }); }); async function cleanupBackups() { diff --git a/src/test/backuptask-test.js b/src/test/backuptask-test.js index 888716efe..95339eff8 100644 --- a/src/test/backuptask-test.js +++ b/src/test/backuptask-test.js @@ -29,8 +29,6 @@ describe('backuptask', function () { provider: 'filesystem', backupFolder: path.join(os.tmpdir(), 'backupstask-test-filesystem'), format: 'tgz', - retentionPolicy: { keepWithinSecs: 10000 }, - schedulePattern: '00 00 23 * * *' }; before(async function () { diff --git a/src/test/settings-test.js b/src/test/settings-test.js index fb77a82a1..52ebfd210 100644 --- a/src/test/settings-test.js +++ b/src/test/settings-test.js @@ -53,17 +53,23 @@ describe('Settings', function () { expect(newBackupConfig.backupFolder).to.be('/tmp/backups'); }); - it('cannot set backup config with invalid schedulePattern', async function () { - let backupConfig = await settings.getBackupConfig(); - backupConfig.schedulePattern = ''; - const [error] = await safe(settings.setBackupConfig(backupConfig)); + it('cannot set backup policy with invalid schedule', async function () { + const [error] = await safe(settings.setBackupPolicy({ schedule: '', retention: { keepWithinSecs: 1 }})); expect(error.reason).to.be(BoxError.BAD_FIELD); }); - it('can set backup config with valid schedulePattern', async function () { - let backupConfig = await settings.getBackupConfig(); - backupConfig.schedulePattern = '00 00 2,23 * * 0,1,2'; - await settings.setBackupConfig(backupConfig); + it('cannot set backup policy with missing retention', async function () { + const [error] = await safe(settings.setBackupPolicy({ schedule: '00 * * * * *'})); + expect(error.reason).to.be(BoxError.BAD_FIELD); + }); + + it('cannot set backup policy with invalid retention', async function () { + const [error] = await safe(settings.setBackupPolicy({ schedule: '00 * * * * *', retention: { keepWhenever: 4 }})); + expect(error.reason).to.be(BoxError.BAD_FIELD); + }); + + it('can set valid backup policy', async function () { + await settings.setBackupPolicy({ schedule: '00 00 2,23 * * 0,1,2', retention: { keepWithinSecs: 1 }}); }); it('can get default unstable apps setting', async function () { diff --git a/src/test/storage-test.js b/src/test/storage-test.js index 2ff99120c..e254687f3 100644 --- a/src/test/storage-test.js +++ b/src/test/storage-test.js @@ -38,8 +38,6 @@ describe('Storage', function () { key: 'key', backupFolder: null, format: 'tgz', - retentionPolicy: { keepWithinSecs: 10000 }, - schedulePattern: '00 00 23 * * *' }; before(function (done) {