diff --git a/migrations/20250724102340-backupTargets-create-table.js b/migrations/20250724102340-backupTargets-create-table.js new file mode 100644 index 000000000..dcea909dd --- /dev/null +++ b/migrations/20250724102340-backupTargets-create-table.js @@ -0,0 +1,65 @@ +'use strict'; + +const paths = require('../src/paths.js'), + uuid = require('uuid'); + +exports.up = async function (db) { + const cmd = 'CREATE TABLE IF NOT EXISTS backupTargets(' + + 'id VARCHAR(128) NOT NULL UNIQUE,' + + 'label VARCHAR(128),' + + 'configJson TEXT,' + + 'limitsJson TEXT,' + + 'retentionJson TEXT,' + + 'encryptionJson TEXT,' + + 'format VARCHAR(16) NOT NULL,' + + 'schedule VARCHAR(128),' + + 'priority BOOLEAN DEFAULT false,' + + 'taskId INTEGER,' + + 'creationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,' + + 'ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,' + + 'FOREIGN KEY(taskId) REFERENCES tasks(id),' + + 'PRIMARY KEY (id)) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin'; + + await db.runSql(cmd); + + const results = await db.runSql('SELECT name, value FROM settings WHERE name=? OR name=? OR name=?', [ 'backup_storage', 'backup_limits', 'backup_policy' ]); + + const label = '', priority = true; + let config = null, limits = null, encryption = null, format = null; + let retention = { keepWithinSecs: 2 * 24 * 60 * 60 }; + let schedule = '00 00 23 * * *';; + + if (results.length === 0) { + config = { provider: 'filesystem', backupFolder: paths.DEFAULT_BACKUP_DIR }; + format = 'tgz'; + } else { + for (const r of results) { + if (r.name === 'backup_storage') { + const tmp = JSON.parse(r.value); + encryption = tmp.encryption || null; + delete tmp.encryption; + + format = tmp.format; + delete tmp.format; + config = tmp; + } else if (r.name === 'backup_limits') { + limits = JSON.parse(r.value); + } else if (r.name === 'backup_policy') { + const tmp = JSON.parse(r.value); + retention = tmp.retention; + schedule = tmp.schedule; + } + } + } + + await db.runSql('START TRANSACTION'); + await db.runSql('INSERT INTO backupTargets (id, label, configJson, limitsJson, retentionJson, schedule, encryptionJson, format, priority) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)', + [ `bc-${uuid.v4()}`, label, JSON.stringify(config), JSON.stringify(limits), JSON.stringify(retention), schedule, JSON.stringify(encryption), format, priority ]); + + await db.runSql('DELETE FROM settings WHERE name=? OR name=? OR name=?', [ 'backup_storage', 'backup_limits', 'backup_policy' ]); + await db.runSql('COMMIT'); +}; + +exports.down = async function (db) { + await db.runSql('DROP TABLE backupTargets'); +}; diff --git a/migrations/schema.sql b/migrations/schema.sql index 7e498ba3a..c4589ea23 100644 --- a/migrations/schema.sql +++ b/migrations/schema.sql @@ -301,6 +301,24 @@ CREATE TABLE IF NOT EXISTS dockerRegistries( PRIMARY KEY (id) ); +CREATE TABLE IF NOT EXISTS backupTargets( + id VARCHAR(128) NOT NULL UNIQUE, + label VARCHAR(128), + configJson TEXT, + limitsJson TEXT, + retentionJson TEXT, + encryptionJson TEXT, + format VARCHAR(16) NOT NULL, + schedule VARCHAR(128), + priority BOOLEAN DEFAULT false, // only because 'default' and 'primary' are reserved keywords + taskId INTEGER, // current task + creationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + + FOREIGN KEY(taskId) REFERENCES tasks(id), + PRIMARY KEY (id) +); + CREATE TABLE IF NOT EXISTS volumes( id VARCHAR(128) NOT NULL UNIQUE, name VARCHAR(256) NOT NULL UNIQUE, diff --git a/src/backupformat.js b/src/backupformat.js index eecb51af8..eca13d9b2 100644 --- a/src/backupformat.js +++ b/src/backupformat.js @@ -4,9 +4,13 @@ exports = module.exports = { api }; +const BoxError = require('./boxerror.js'); + function api(format) { switch (format) { case 'tgz': return require('./backupformat/tgz.js'); case 'rsync': return require('./backupformat/rsync.js'); } + + throw new BoxError(BoxError.INTERNAL_ERROR, `Undefined format ${format}`); } diff --git a/src/backups.js b/src/backups.js index daa0be706..1150b24a8 100644 --- a/src/backups.js +++ b/src/backups.js @@ -41,6 +41,8 @@ exports = module.exports = { getMountStatus, ensureMounted, + _addDefaultTarget: addDefaultTarget, + BACKUP_IDENTIFIER_BOX: 'box', BACKUP_IDENTIFIER_MAIL: 'mail', @@ -71,9 +73,12 @@ const assert = require('assert'), settings = require('./settings.js'), storage = require('./storage.js'), tasks = require('./tasks.js'), + uuid = require('uuid'), _ = require('./underscore.js'); -const BACKUPS_FIELDS = [ 'id', 'remotePath', 'label', 'identifier', 'creationTime', 'packageVersion', 'type', 'dependsOnJson', 'state', 'manifestJson', 'format', 'preserveSecs', 'encryptionVersion', 'appConfigJson' ]; +const BACKUPS_FIELDS = [ 'id', 'remotePath', 'label', 'identifier', 'creationTime', 'packageVersion', 'type', 'dependsOnJson', 'state', 'manifestJson', 'format', 'preserveSecs', 'encryptionVersion', 'appConfigJson' ].join(','); + +const BACKUP_TARGET_FIELDS = [ 'id', 'label', 'configJson', 'limitsJson', 'retentionJson', 'schedule', 'encryptionJson', 'format', 'priority', 'creationTime', 'ts' ].join(','); function postProcess(result) { assert.strictEqual(typeof result, 'object'); @@ -90,6 +95,26 @@ function postProcess(result) { return result; } +function postProcessTarget(result) { + assert.strictEqual(typeof result, 'object'); + + result.config = result.configJson ? safe.JSON.parse(result.configJson) : {}; + delete result.configJson; + + result.limits = result.limitsJson ? safe.JSON.parse(result.limitsJson) : {}; + delete result.limitsJson; + + result.retention = result.retentionJson ? safe.JSON.parse(result.retentionJson) : {}; + delete result.retentionJson; + + result.encryption = result.encryptionJson ? safe.JSON.parse(result.encryptionJson) : null; + delete result.encryptionJson; + + result.priority = !!result.priority; + + return result; +} + function removePrivateFields(backupConfig) { assert.strictEqual(typeof backupConfig, 'object'); if (backupConfig.encryption) { @@ -403,11 +428,9 @@ async function ensureMounted() { } async function getPolicy() { - const result = await settings.getJson(settings.BACKUP_POLICY_KEY); - return result || { - retention: { keepWithinSecs: 2 * 24 * 60 * 60 }, // 2 days - schedule: '00 00 23 * * *' // every day at 11pm - }; + const results = await database.query(`SELECT ${BACKUP_TARGET_FIELDS} FROM backupTargets WHERE priority=?`, [ true ]); + const result = postProcessTarget(results[0]); + return { retention: result.retention, schedule: result.schedule }; } async function setPolicy(policy) { @@ -416,7 +439,8 @@ async function setPolicy(policy) { const error = await validatePolicy(policy); if (error) throw error; - await settings.setJson(settings.BACKUP_POLICY_KEY, policy); + await updateTarget(policy); + await cron.handleBackupPolicyChanged(policy); } @@ -435,19 +459,60 @@ function getRootPath(storageConfig, mountPath) { } } +async function updateTarget(data) { + assert(data && typeof data === 'object'); + + const args = []; + const fields = []; + for (const k in data) { + if (k === 'label' || k === 'schedule' || k === 'format' || k === 'priority') { + fields.push(k + ' = ?'); + args.push(data[k]); + } else if (k === 'config' || k === 'limits' || k === 'retention' || k === 'encryption') { + fields.push(`${k}JSON = ?`); + args.push(JSON.stringify(data[k])); + } + } + args.push(true); // primary flag + + const [updateError, result] = await safe(database.query('UPDATE backupTargets SET ' + fields.join(', ') + ' WHERE priority = ?', args)); + if (updateError) throw updateError; + if (result.affectedRows !== 1) throw new BoxError(BoxError.NOT_FOUND, 'Target not found'); +} + +async function addDefaultTarget() { + const label = '', priority = true; + const limits = null, encryption = null; + const retention = { keepWithinSecs: 2 * 24 * 60 * 60 }; + const schedule = '00 00 23 * * *';; + const config = { provider: 'filesystem', backupFolder: paths.DEFAULT_BACKUP_DIR }; + const format = 'tgz'; + + await database.query('INSERT INTO backupTargets (id, label, configJson, limitsJson, retentionJson, schedule, encryptionJson, format, priority) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)', + [ `bc-${uuid.v4()}`, label, JSON.stringify(config), JSON.stringify(limits), JSON.stringify(retention), schedule, JSON.stringify(encryption), format, priority ]); +} + async function getConfig() { - const result = await settings.getJson(settings.BACKUP_STORAGE_KEY) || { provider: 'filesystem', backupFolder: paths.DEFAULT_BACKUP_DIR, format: 'tgz', encryption: null }; - const limits = await settings.getJson(settings.BACKUP_LIMITS_KEY); - if (limits) result.limits = limits; - result.rootPath = getRootPath(result, paths.MANAGED_BACKUP_MOUNT_DIR); // note: rootPath will be dynamic for managed mount providers during app import - return result; + const results = await database.query(`SELECT ${BACKUP_TARGET_FIELDS} FROM backupTargets WHERE priority=?`, [ true ]); + const result = postProcessTarget(results[0]); + + const config = result.config; + config.format = result.format; + config.encryption = result.encryption; + config.rootPath = getRootPath(config, paths.MANAGED_BACKUP_MOUNT_DIR); // note: rootPath will be dynamic for managed mount providers during app import + + return config; } async function setConfig(backupConfig) { assert.strictEqual(typeof backupConfig, 'object'); - await settings.setJson(settings.BACKUP_STORAGE_KEY, _.omit(backupConfig, ['limits'])); - await settings.setJson(settings.BACKUP_LIMITS_KEY, backupConfig.limits || null); + await updateTarget({ + config: _.omit(backupConfig, ['limits', 'format', 'encryption']), + format: backupConfig.format, + encryption: backupConfig.encryption || null, + limits: backupConfig.limits || null + }); } async function setLimits(limits) { @@ -529,5 +594,5 @@ async function setStorage(storageConfig) { debug('setStorage: clearing backup cache'); cleanupCacheFilesSync(); - await settings.setJson(settings.BACKUP_STORAGE_KEY, storageConfig); + await setConfig(storageConfig); } diff --git a/src/routes/test/common.js b/src/routes/test/common.js index c18e9ae06..e7ccf03b8 100644 --- a/src/routes/test/common.js +++ b/src/routes/test/common.js @@ -2,6 +2,7 @@ const apps = require('../../apps.js'), appstore = require('../../appstore.js'), + backups = require('../../backups.js'), debug = require('debug')('box:test/common'), constants = require('../../constants.js'), database = require('../../database.js'), @@ -114,6 +115,7 @@ async function setupServer() { await database.initialize(); await database._clear(); await appstore._setApiServerOrigin(exports.mockApiServerOrigin); + await backups._addDefaultTarget(); await oidcServer.stop(); await server.start(); debug('Set up server complete'); diff --git a/src/test/common.js b/src/test/common.js index 12cdf4202..7d011226e 100644 --- a/src/test/common.js +++ b/src/test/common.js @@ -2,6 +2,7 @@ const apps = require('../apps.js'), appstore = require('../appstore.js'), + backups = require('../backups.js'), constants = require('../constants.js'), cron = require('../cron.js'), dashboard = require('../dashboard.js'), @@ -217,6 +218,7 @@ async function databaseSetup() { await database._clear(); await appstore._setApiServerOrigin(exports.mockApiServerOrigin); await dashboard._setLocation(constants.DASHBOARD_SUBDOMAIN, exports.dashboardDomain); + await backups._addDefaultTarget(); } async function domainSetup() {