jjkiers/cloudron-box
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

rename backups to backuptargets

Browse Source
This commit is contained in:
Girish Ramakrishnan
2025-07-24 18:46:21 +02:00
parent 5e456f378b
commit 931311f11f
20 changed files with 104 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/backups.js
-450
View File
@@ -1,450 +0,0 @@
'use strict';
exports = module.exports = {
startBackupTask,
startCleanupTask,
cleanupCacheFilesSync,
removePrivateFields,
generateEncryptionKeysSync,
getSnapshotInfo,
setSnapshotInfo,
validatePolicy,
testStorage,
validateFormat,
getPolicy,
setPolicy,
getTarget,
getConfig,
setConfig,
setStorage,
setLimits,
getRootPath,
setupManagedStorage,
remount,
getMountStatus,
ensureMounted,
_addDefaultTarget: addDefaultTarget,
BACKUP_IDENTIFIER_BOX: 'box',
BACKUP_IDENTIFIER_MAIL: 'mail',
BACKUP_TYPE_APP: 'app',
BACKUP_TYPE_BOX: 'box',
BACKUP_TYPE_MAIL: 'mail',
BACKUP_STATE_NORMAL: 'normal', // should rename to created to avoid listing in UI?
BACKUP_STATE_CREATING: 'creating',
BACKUP_STATE_ERROR: 'error',
};
const assert = require('assert'),
backupListing = require('./backuplisting.js'),
BoxError = require('./boxerror.js'),
constants = require('./constants.js'),
cron = require('./cron.js'),
{ CronTime } = require('cron'),
crypto = require('crypto'),
database = require('./database.js'),
debug = require('debug')('box:backups'),
eventlog = require('./eventlog.js'),
locks = require('./locks.js'),
mounts = require('./mounts.js'),
path = require('path'),
paths = require('./paths.js'),
safe = require('safetydance'),
settings = require('./settings.js'),
storage = require('./storage.js'),
tasks = require('./tasks.js'),
uuid = require('uuid'),
_ = require('./underscore.js');
const BACKUP_TARGET_FIELDS = [ 'id', 'label', 'configJson', 'limitsJson', 'retentionJson', 'schedule', 'encryptionJson', 'format', 'priority', 'creationTime', 'ts' ].join(',');
function postProcess(result) {
assert.strictEqual(typeof result, 'object');
result.config = result.configJson ? safe.JSON.parse(result.configJson) : {};
delete result.configJson;
result.limits = result.limitsJson ? safe.JSON.parse(result.limitsJson) : {};
delete result.limitsJson;
result.retention = result.retentionJson ? safe.JSON.parse(result.retentionJson) : {};
delete result.retentionJson;
result.encryption = result.encryptionJson ? safe.JSON.parse(result.encryptionJson) : null;
delete result.encryptionJson;
result.priority = !!result.priority;
return result;
}
function removePrivateFields(backupConfig) {
assert.strictEqual(typeof backupConfig, 'object');
if (backupConfig.encryption) {
delete backupConfig.encryption;
backupConfig.password = constants.SECRET_PLACEHOLDER;
}
delete backupConfig.rootPath;
return storage.api(backupConfig.provider).removePrivateFields(backupConfig);
}
// this function is used in migrations - 20200512172301-settings-backup-encryption.js
function generateEncryptionKeysSync(password) {
assert.strictEqual(typeof password, 'string');
const aesKeys = crypto.scryptSync(password, Buffer.from('CLOUDRONSCRYPTSALT', 'utf8'), 128);
return {
dataKey: aesKeys.subarray(0, 32).toString('hex'),
dataHmacKey: aesKeys.subarray(32, 64).toString('hex'),
filenameKey: aesKeys.subarray(64, 96).toString('hex'),
filenameHmacKey: aesKeys.subarray(96).toString('hex')
};
}
async function validatePolicy(policy) {
assert.strictEqual(typeof policy, 'object');
const job = safe.safeCall(function () { return new CronTime(policy.schedule); });
if (!job) return new BoxError(BoxError.BAD_FIELD, 'Invalid schedule pattern');
const retention = policy.retention;
if (!retention) return new BoxError(BoxError.BAD_FIELD, 'retention is required');
if (!['keepWithinSecs','keepDaily','keepWeekly','keepMonthly','keepYearly'].find(k => !!retention[k])) return new BoxError(BoxError.BAD_FIELD, 'retention properties missing');
if ('keepWithinSecs' in retention && typeof retention.keepWithinSecs !== 'number') return new BoxError(BoxError.BAD_FIELD, 'retention.keepWithinSecs must be a number');
if ('keepDaily' in retention && typeof retention.keepDaily !== 'number') return new BoxError(BoxError.BAD_FIELD, 'retention.keepDaily must be a number');
if ('keepWeekly' in retention && typeof retention.keepWeekly !== 'number') return new BoxError(BoxError.BAD_FIELD, 'retention.keepWeekly must be a number');
if ('keepMonthly' in retention && typeof retention.keepMonthly !== 'number') return new BoxError(BoxError.BAD_FIELD, 'retention.keepMonthly must be a number');
if ('keepYearly' in retention && typeof retention.keepYearly !== 'number') return new BoxError(BoxError.BAD_FIELD, 'retention.keepYearly must be a number');
}
async function startBackupTask(auditSource) {
const [error] = await safe(locks.acquire(locks.TYPE_FULL_BACKUP_TASK));
if (error) throw new BoxError(BoxError.BAD_STATE, `Another backup task is in progress: ${error.message}`);
const backupConfig = await getConfig();
const memoryLimit = backupConfig.limits?.memoryLimit ? Math.max(backupConfig.limits.memoryLimit/1024/1024, 1024) : 1024;
const taskId = await tasks.add(tasks.TASK_BACKUP, [ { /* options */ } ]);
await eventlog.add(eventlog.ACTION_BACKUP_START, auditSource, { taskId });
// background
tasks.startTask(taskId, { timeout: 24 * 60 * 60 * 1000 /* 24 hours */, nice: 15, memoryLimit, oomScoreAdjust: -999 })
.then(async (backupId) => {
const backup = await backupListing.get(backupId);
await eventlog.add(eventlog.ACTION_BACKUP_FINISH, auditSource, { taskId, backupId, remotePath: backup.remotePath });
})
.catch(async (error) => {
const timedOut = error.code === tasks.ETIMEOUT;
await safe(eventlog.add(eventlog.ACTION_BACKUP_FINISH, auditSource, { taskId, errorMessage: error.message, timedOut }));
})
.finally(async () => {
await locks.release(locks.TYPE_FULL_BACKUP_TASK);
await locks.releaseByTaskId(taskId);
});
return taskId;
}
// this function is used in migrations - 20200512172301-settings-backup-encryption.js
function cleanupCacheFilesSync() {
const files = safe.fs.readdirSync(path.join(paths.BACKUP_INFO_DIR));
if (!files) return;
files
.filter(function (f) { return f.endsWith('.sync.cache'); })
.forEach(function (f) {
safe.fs.unlinkSync(path.join(paths.BACKUP_INFO_DIR, f));
});
}
function getSnapshotInfo(id) {
assert.strictEqual(typeof id, 'string');
const contents = safe.fs.readFileSync(paths.SNAPSHOT_INFO_FILE, 'utf8');
const info = safe.JSON.parse(contents);
if (!info) return { };
return info[id] || { };
}
// keeps track of contents of the snapshot directory. this provides a way to clean up backups of uninstalled apps
async function setSnapshotInfo(id, info) {
assert.strictEqual(typeof id, 'string');
assert.strictEqual(typeof info, 'object');
const contents = safe.fs.readFileSync(paths.SNAPSHOT_INFO_FILE, 'utf8');
const data = safe.JSON.parse(contents) || { };
if (info) data[id] = info; else delete data[id];
if (!safe.fs.writeFileSync(paths.SNAPSHOT_INFO_FILE, JSON.stringify(data, null, 4), 'utf8')) {
throw new BoxError(BoxError.FS_ERROR, safe.error.message);
}
}
async function startCleanupTask(auditSource) {
assert.strictEqual(typeof auditSource, 'object');
const taskId = await tasks.add(tasks.TASK_CLEAN_BACKUPS, []);
// background
tasks.startTask(taskId, {})
.then(async (result) => { // { removedBoxBackupPaths, removedAppBackupPaths, removedMailBackupPaths, missingBackupPaths }
await eventlog.add(eventlog.ACTION_BACKUP_CLEANUP_FINISH, auditSource, { taskId, errorMessage: null, ...result });
})
.catch(async (error) => {
await eventlog.add(eventlog.ACTION_BACKUP_CLEANUP_FINISH, auditSource, { taskId, errorMessage: error.message });
});
return taskId;
}
async function testStorage(storageConfig) {
assert.strictEqual(typeof storageConfig, 'object');
const func = storage.api(storageConfig.provider);
if (!func) return new BoxError(BoxError.BAD_FIELD, 'unknown storage provider');
await storage.api(storageConfig.provider).testConfig(storageConfig);
}
function validateEncryptionPassword(password) {
assert.strictEqual(typeof password, 'string');
if (password.length < 8) return new BoxError(BoxError.BAD_FIELD, 'password must be atleast 8 characters');
}
function managedBackupMountObject(backupConfig) {
assert(mounts.isManagedProvider(backupConfig.provider));
return {
name: 'backup',
hostPath: paths.MANAGED_BACKUP_MOUNT_DIR,
mountType: backupConfig.provider,
mountOptions: backupConfig.mountOptions
};
}
async function remount() {
const backupConfig = await getConfig();
if (mounts.isManagedProvider(backupConfig.provider)) {
await mounts.remount(managedBackupMountObject(backupConfig));
}
}
async function getMountStatus() {
const backupConfig = await getConfig();
let hostPath;
if (mounts.isManagedProvider(backupConfig.provider)) {
hostPath = paths.MANAGED_BACKUP_MOUNT_DIR;
} else if (backupConfig.provider === 'mountpoint') {
hostPath = backupConfig.mountPoint;
} else if (backupConfig.provider === 'filesystem') {
hostPath = backupConfig.backupFolder;
} else {
return { state: 'active' };
}
return await mounts.getStatus(backupConfig.provider, hostPath); // { state, message }
}
async function ensureMounted() {
const status = await getMountStatus();
if (status.state === 'active') return status;
await remount();
return await getMountStatus();
}
async function getPolicy() {
const results = await database.query(`SELECT ${BACKUP_TARGET_FIELDS} FROM backupTargets WHERE priority=?`, [ true ]);
const result = postProcess(results[0]);
return { retention: result.retention, schedule: result.schedule };
}
async function setPolicy(policy) {
assert.strictEqual(typeof policy, 'object');
const error = await validatePolicy(policy);
if (error) throw error;
await updateTarget(policy);
await cron.handleBackupPolicyChanged(policy);
}
function getRootPath(storageConfig, mountPath) {
assert.strictEqual(typeof storageConfig, 'object');
assert.strictEqual(typeof mountPath, 'string');
if (mounts.isManagedProvider(storageConfig.provider)) {
return path.join(mountPath, storageConfig.prefix);
} else if (storageConfig.provider === 'mountpoint') {
return path.join(storageConfig.mountPoint, storageConfig.prefix);
} else if (storageConfig.provider === 'filesystem') {
return storageConfig.backupFolder;
} else {
return storageConfig.prefix;
}
}
async function updateTarget(data) {
assert(data && typeof data === 'object');
const args = [];
const fields = [];
for (const k in data) {
if (k === 'label' || k === 'schedule' || k === 'format' || k === 'priority') {
fields.push(k + ' = ?');
args.push(data[k]);
} else if (k === 'config' || k === 'limits' || k === 'retention' || k === 'encryption') {
fields.push(`${k}JSON = ?`);
args.push(JSON.stringify(data[k]));
}
}
args.push(true); // primary flag
const [updateError, result] = await safe(database.query('UPDATE backupTargets SET ' + fields.join(', ') + ' WHERE priority = ?', args));
if (updateError) throw updateError;
if (result.affectedRows !== 1) throw new BoxError(BoxError.NOT_FOUND, 'Target not found');
}
async function addDefaultTarget() {
const label = '', priority = true;
const limits = null, encryption = null;
const retention = { keepWithinSecs: 2 * 24 * 60 * 60 };
const schedule = '00 00 23 * * *';;
const config = { provider: 'filesystem', backupFolder: paths.DEFAULT_BACKUP_DIR };
const format = 'tgz';
const id = `bc-${uuid.v4()}`;
await database.query('INSERT INTO backupTargets (id, label, configJson, limitsJson, retentionJson, schedule, encryptionJson, format, priority) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)',
[ id, label, JSON.stringify(config), JSON.stringify(limits), JSON.stringify(retention), schedule, JSON.stringify(encryption), format, priority ]);
return id;
}
async function getTarget(id) {
const results = await database.query(`SELECT ${BACKUP_TARGET_FIELDS} FROM backupTargets WHERE id=?`, [ id ]);
if (results.length === 0) return null;
return postProcess(results[0]);
}
async function getConfig() {
const results = await database.query(`SELECT ${BACKUP_TARGET_FIELDS} FROM backupTargets WHERE priority=?`, [ true ]);
const result = postProcess(results[0]);
const config = result.config;
config.format = result.format;
config.encryption = result.encryption;
config.rootPath = getRootPath(config, paths.MANAGED_BACKUP_MOUNT_DIR); // note: rootPath will be dynamic for managed mount providers during app import
return config;
}
async function setConfig(backupConfig) {
assert.strictEqual(typeof backupConfig, 'object');
await updateTarget({
config: _.omit(backupConfig, ['limits', 'format', 'encryption']),
format: backupConfig.format,
encryption: backupConfig.encryption || null,
limits: backupConfig.limits || null
});
}
async function setLimits(limits) {
assert.strictEqual(typeof limits, 'object');
await settings.setJson(settings.BACKUP_LIMITS_KEY, limits);
}
function validateFormat(format) {
assert.strictEqual(typeof format, 'string');
if (format === 'tgz' || format == 'rsync') return null;
return new BoxError(BoxError.BAD_FIELD, 'Invalid backup format');
}
async function setupManagedStorage(storageConfig, hostPath) {
assert.strictEqual(typeof storageConfig, 'object');
assert.strictEqual(typeof hostPath, 'string');
if (!mounts.isManagedProvider(storageConfig.provider)) return null;
if (!storageConfig.mountOptions || typeof storageConfig.mountOptions !== 'object') throw new BoxError(BoxError.BAD_FIELD, 'mountOptions must be an object');
const error = mounts.validateMountOptions(storageConfig.provider, storageConfig.mountOptions);
if (error) throw error;
debug(`setupManagedStorage: setting up mount at ${hostPath} with ${storageConfig.provider}`);
const newMount = {
name: path.basename(hostPath),
hostPath,
mountType: storageConfig.provider,
mountOptions: storageConfig.mountOptions
};
await mounts.tryAddMount(newMount, { timeout: 10 }); // 10 seconds
return newMount;
}
async function setStorage(storageConfig) {
assert.strictEqual(typeof storageConfig, 'object');
if (constants.DEMO) throw new BoxError(BoxError.BAD_STATE, 'Not allowed in demo mode');
const oldConfig = await getConfig();
if (storageConfig.provider === oldConfig.provider) storage.api(storageConfig.provider).injectPrivateFields(storageConfig, oldConfig);
const formatError = validateFormat(storageConfig.format);
if (formatError) throw formatError;
storageConfig.encryption = null;
if ('password' in storageConfig) { // user set password
if (storageConfig.password === constants.SECRET_PLACEHOLDER) {
storageConfig.encryption = oldConfig.encryption || null;
} else {
const encryptionPasswordError = validateEncryptionPassword(storageConfig.password);
if (encryptionPasswordError) throw encryptionPasswordError;
storageConfig.encryption = generateEncryptionKeysSync(storageConfig.password);
}
delete storageConfig.password;
}
debug('setStorage: validating new storage configuration');
const testMountObject = await setupManagedStorage(storageConfig, '/mnt/backup-storage-validation'); // this validates mountOptions
const testStorageError = await testStorage(Object.assign({ mountPath: '/mnt/backup-storage-validation' }, storageConfig)); // this validates provider and it's api options. requires mountPath
if (testMountObject) await mounts.removeMount(testMountObject);
if (testStorageError) throw testStorageError;
debug('setStorage: removing old storage configuration');
if (mounts.isManagedProvider(oldConfig.provider)) await safe(mounts.removeMount(managedBackupMountObject(oldConfig)));
debug('setStorage: setting up new storage configuration');
await setupManagedStorage(storageConfig, paths.MANAGED_BACKUP_MOUNT_DIR);
debug('setStorage: clearing backup cache');
cleanupCacheFilesSync();
await setConfig(storageConfig);
}