jjkiers/cloudron-box
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

backups: make id, provider a backend specific setting

Browse Source 
the backend can stash whatever values it wants in the config.
just like the DNS backends, we make verifyConfig return a sanitized config.
added benefit is that extra user fields (via API) are also not dumped into the db.
This commit is contained in:
Girish Ramakrishnan
2025-08-01 18:55:04 +02:00
parent 9dfe6242b9
commit 53e9925880
7 changed files with 165 additions and 134 deletions
Download Patch File Download Diff File Expand all files Collapse all files

170
src/storage/filesystem.js
View File

@@ -5,7 +5,7 @@ exports = module.exports = {
teardown,
cleanup,
testConfig,
verifyConfig,
removePrivateFields,
injectPrivateFields,
@@ -33,20 +33,21 @@ const assert = require('assert'),
path = require('path'),
paths = require('../paths.js'),
safe = require('safetydance'),
shell = require('../shell.js')('filesystem');
shell = require('../shell.js')('filesystem'),
_ = require('../underscore.js');
async function getAvailableSize(apiConfig) {
assert.strictEqual(typeof apiConfig, 'object');
async function getAvailableSize(config) {
assert.strictEqual(typeof config, 'object');
// note that df returns the disk size (as opposed to the apparent size)
const [error, dfResult] = await safe(df.file(apiConfig.rootPath));
const [error, dfResult] = await safe(df.file(config.rootPath));
if (error) throw new BoxError(BoxError.FS_ERROR, `Error when checking for disk space: ${error.message}`);
return dfResult.available;
}
function hasChownSupportSync(apiConfig) {
switch (apiConfig.provider) {
function hasChownSupportSync(config) {
switch (config.provider) {
case mounts.MOUNT_TYPE_NFS:
case mounts.MOUNT_TYPE_EXT4:
case mounts.MOUNT_TYPE_XFS:
@@ -56,16 +57,16 @@ function hasChownSupportSync(apiConfig) {
case mounts.MOUNT_TYPE_SSHFS:
// sshfs can be mounted as root or normal user. when mounted as root, we have to chown since we remove backups as the yellowtent user
// when mounted as non-root user, files are created as yellowtent user but they are still owned by the non-root user (thus del also works)
return apiConfig.mountOptions.user === 'root';
return config.mountOptions.user === 'root';
case mounts.MOUNT_TYPE_CIFS:
return true;
case mounts.MOUNT_TYPE_MOUNTPOINT:
return apiConfig.chown;
return config.chown;
}
}
async function upload(apiConfig, backupFilePath) {
assert.strictEqual(typeof apiConfig, 'object');
async function upload(config, backupFilePath) {
assert.strictEqual(typeof config, 'object');
assert.strictEqual(typeof backupFilePath, 'string');
const [mkdirError] = await safe(fs.promises.mkdir(path.dirname(backupFilePath), { recursive: true }));
@@ -77,7 +78,7 @@ async function upload(apiConfig, backupFilePath) {
stream: fs.createWriteStream(backupFilePath, { autoClose: true }),
async finish() {
const backupUid = parseInt(process.env.SUDO_UID, 10) || process.getuid(); // in test, upload() may or may not be called via sudo script
if (hasChownSupportSync(apiConfig)) {
if (hasChownSupportSync(config)) {
if (!safe.fs.chownSync(backupFilePath, backupUid, backupUid)) throw new BoxError(BoxError.EXTERNAL_ERROR, `Unable to chown ${backupFilePath}: ${safe.error.message}`);
if (!safe.fs.chownSync(path.dirname(backupFilePath), backupUid, backupUid)) throw new BoxError(BoxError.EXTERNAL_ERROR, `Unable to chown parentdir ${backupFilePath}: ${safe.error.message}`);
}
@@ -85,8 +86,8 @@ async function upload(apiConfig, backupFilePath) {
};
}
async function download(apiConfig, sourceFilePath) {
assert.strictEqual(typeof apiConfig, 'object');
async function download(config, sourceFilePath) {
assert.strictEqual(typeof config, 'object');
assert.strictEqual(typeof sourceFilePath, 'string');
debug(`download: ${sourceFilePath}`);
@@ -96,8 +97,8 @@ async function download(apiConfig, sourceFilePath) {
return fs.createReadStream(sourceFilePath);
}
async function exists(apiConfig, sourceFilePath) {
assert.strictEqual(typeof apiConfig, 'object');
async function exists(config, sourceFilePath) {
assert.strictEqual(typeof config, 'object');
assert.strictEqual(typeof sourceFilePath, 'string');
// do not use existsSync because it does not return EPERM etc
@@ -109,8 +110,8 @@ async function exists(apiConfig, sourceFilePath) {
return true;
}
async function listDir(apiConfig, dir, batchSize, marker) {
assert.strictEqual(typeof apiConfig, 'object');
async function listDir(config, dir, batchSize, marker) {
assert.strictEqual(typeof config, 'object');
assert.strictEqual(typeof dir, 'string');
assert.strictEqual(typeof batchSize, 'number');
assert(typeof marker !== 'undefined');
@@ -142,8 +143,8 @@ async function listDir(apiConfig, dir, batchSize, marker) {
return { entries: fileStream.splice(0, batchSize), marker }; // note: splice also modifies the array
}
async function copy(apiConfig, oldFilePath, newFilePath, progressCallback) {
assert.strictEqual(typeof apiConfig, 'object');
async function copy(config, oldFilePath, newFilePath, progressCallback) {
assert.strictEqual(typeof config, 'object');
assert.strictEqual(typeof oldFilePath, 'string');
assert.strictEqual(typeof newFilePath, 'string');
assert.strictEqual(typeof progressCallback, 'function');
@@ -153,13 +154,13 @@ async function copy(apiConfig, oldFilePath, newFilePath, progressCallback) {
progressCallback({ message: `Copying ${oldFilePath} to ${newFilePath}` });
let cpOptions = ((apiConfig.provider !== mounts.MOUNT_TYPE_MOUNTPOINT && apiConfig.provider !== mounts.MOUNT_TYPE_CIFS) || apiConfig.preserveAttributes) ? '-a' : '-dR';
cpOptions += apiConfig.noHardlinks ? '' : 'l'; // this will hardlink backups saving space
let cpOptions = ((config.provider !== mounts.MOUNT_TYPE_MOUNTPOINT && config.provider !== mounts.MOUNT_TYPE_CIFS) || config.preserveAttributes) ? '-a' : '-dR';
cpOptions += config.noHardlinks ? '' : 'l'; // this will hardlink backups saving space
if (apiConfig.provider === mounts.MOUNT_TYPE_SSHFS) {
const identityFilePath = path.join(paths.SSHFS_KEYS_DIR, `id_rsa_${apiConfig.mountOptions.host}`);
if (config.provider === mounts.MOUNT_TYPE_SSHFS) {
const identityFilePath = path.join(paths.SSHFS_KEYS_DIR, `id_rsa_${config.mountOptions.host}`);
const sshOptions = [ '-o', '"StrictHostKeyChecking no"', '-i', identityFilePath, '-p', apiConfig.mountOptions.port, `${apiConfig.mountOptions.user}@${apiConfig.mountOptions.host}` ];
const sshOptions = [ '-o', '"StrictHostKeyChecking no"', '-i', identityFilePath, '-p', config.mountOptions.port, `${config.mountOptions.user}@${config.mountOptions.host}` ];
const sshArgs = sshOptions.concat([ 'cp', cpOptions, oldFilePath.replace('/mnt/cloudronbackup/', ''), newFilePath.replace('/mnt/cloudronbackup/', '') ]);
const [remoteCopyError] = await safe(shell.spawn('ssh', sshArgs, { shell: true }));
if (!remoteCopyError) return;
@@ -171,8 +172,8 @@ async function copy(apiConfig, oldFilePath, newFilePath, progressCallback) {
if (copyError) throw new BoxError(BoxError.EXTERNAL_ERROR, copyError.message);
}
async function remove(apiConfig, filename) {
assert.strictEqual(typeof apiConfig, 'object');
async function remove(config, filename) {
assert.strictEqual(typeof config, 'object');
assert.strictEqual(typeof filename, 'string');
const stat = safe.fs.statSync(filename);
@@ -185,17 +186,17 @@ async function remove(apiConfig, filename) {
}
}
async function removeDir(apiConfig, pathPrefix, progressCallback) {
assert.strictEqual(typeof apiConfig, 'object');
async function removeDir(config, pathPrefix, progressCallback) {
assert.strictEqual(typeof config, 'object');
assert.strictEqual(typeof pathPrefix, 'string');
assert.strictEqual(typeof progressCallback, 'function');
progressCallback({ message: `Removing directory ${pathPrefix}` });
if (apiConfig.provider === mounts.MOUNT_TYPE_SSHFS) {
const identityFilePath = path.join(paths.SSHFS_KEYS_DIR, `id_rsa_${apiConfig.mountOptions.host}`);
if (config.provider === mounts.MOUNT_TYPE_SSHFS) {
const identityFilePath = path.join(paths.SSHFS_KEYS_DIR, `id_rsa_${config.mountOptions.host}`);
const sshOptions = [ '-o', '"StrictHostKeyChecking no"', '-i', identityFilePath, '-p', apiConfig.mountOptions.port, `${apiConfig.mountOptions.user}@${apiConfig.mountOptions.host}` ];
const sshOptions = [ '-o', '"StrictHostKeyChecking no"', '-i', identityFilePath, '-p', config.mountOptions.port, `${config.mountOptions.user}@${config.mountOptions.host}` ];
const sshArgs = sshOptions.concat([ 'rm', '-rf', pathPrefix.replace('/mnt/cloudronbackup/', '') ]);
const [remoteRmError] = await safe(shell.spawn('ssh', sshArgs, { shell: true }));
if (!remoteRmError) return;
@@ -223,28 +224,23 @@ function validateDestPath(folder) {
return null;
}
async function cleanup(apiConfig, progressCallback) {
assert.strictEqual(typeof apiConfig, 'object');
async function cleanup(config, progressCallback) {
assert.strictEqual(typeof config, 'object');
assert.strictEqual(typeof progressCallback, 'function');
}
async function setupManagedMount(apiConfig, hostPath) {
assert.strictEqual(typeof apiConfig, 'object');
async function setupManagedMount(provider, mountOptions, hostPath) {
assert.strictEqual(typeof provider, 'string');
assert.strictEqual(typeof mountOptions, 'object');
assert.strictEqual(typeof hostPath, 'string');
assert(mounts.isManagedProvider(apiConfig.provider));
if (!apiConfig.mountOptions || typeof apiConfig.mountOptions !== 'object') throw new BoxError(BoxError.BAD_FIELD, 'mountOptions must be an object');
const error = mounts.validateMountOptions(apiConfig.provider, apiConfig.mountOptions);
if (error) throw error;
debug(`setupManagedMount: setting up mount at ${hostPath} with ${apiConfig.provider}`);
debug(`setupManagedMount: setting up mount at ${hostPath} with ${provider}`);
const newMount = {
description: `Cloudron Managed Mount`,
hostPath,
mountType: apiConfig.provider,
mountOptions: apiConfig.mountOptions
mountType: provider,
mountOptions
};
await mounts.tryAddMount(newMount, { timeout: 10 }); // 10 seconds
@@ -252,59 +248,66 @@ async function setupManagedMount(apiConfig, hostPath) {
return newMount;
}
async function setup(apiConfig) {
assert.strictEqual(typeof apiConfig, 'object');
async function setup(config) {
assert.strictEqual(typeof config, 'object');
debug('setup: removing old storage configuration');
if (!mounts.isManagedProvider(apiConfig.provider)) return;
if (!mounts.isManagedProvider(config.provider)) return;
await safe(mounts.removeMount(paths.MANAGED_BACKUP_MOUNT_DIR), { debug }); // ignore error
debug('setup: setting up new storage configuration');
await setupManagedMount(apiConfig, paths.MANAGED_BACKUP_MOUNT_DIR);
await setupManagedMount(config.provider, config.mountOptions, paths.MANAGED_BACKUP_MOUNT_DIR);
}
async function teardown(apiConfig) {
assert.strictEqual(typeof apiConfig, 'object');
async function teardown(config) {
assert.strictEqual(typeof config, 'object');
if (!mounts.isManagedProvider(apiConfig.provider)) return;
if (!mounts.isManagedProvider(config.provider)) return;
await safe(mounts.removeMount(paths.MANAGED_BACKUP_MOUNT_DIR), { debug }); // ignore error
}
async function testConfig(apiConfig) {
assert.strictEqual(typeof apiConfig, 'object');
async function verifyConfig({ id, provider, config }) {
assert.strictEqual(typeof id, 'string');
assert.strictEqual(typeof provider, 'string');
assert.strictEqual(typeof config, 'object');
if ('noHardlinks' in apiConfig && typeof apiConfig.noHardlinks !== 'boolean') throw new BoxError(BoxError.BAD_FIELD, 'noHardlinks must be boolean');
if ('chown' in apiConfig && typeof apiConfig.chown !== 'boolean') throw new BoxError(BoxError.BAD_FIELD, 'chown must be boolean');
if ('preserveAttributes' in apiConfig && typeof apiConfig.preserveAttributes !== 'boolean') throw new BoxError(BoxError.BAD_FIELD, 'preserveAttributes must be boolean');
if ('noHardlinks' in config && typeof config.noHardlinks !== 'boolean') throw new BoxError(BoxError.BAD_FIELD, 'noHardlinks must be boolean');
if ('chown' in config && typeof config.chown !== 'boolean') throw new BoxError(BoxError.BAD_FIELD, 'chown must be boolean');
if ('preserveAttributes' in config && typeof config.preserveAttributes !== 'boolean') throw new BoxError(BoxError.BAD_FIELD, 'preserveAttributes must be boolean');
let rootPath, testMountObject;
if (apiConfig.provider === mounts.MOUNT_TYPE_FILESYSTEM) {
if (!apiConfig.backupFolder || typeof apiConfig.backupFolder !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'backupFolder must be non-empty string');
const error = validateDestPath(apiConfig.backupFolder);
if (provider === mounts.MOUNT_TYPE_FILESYSTEM) {
if (!config.backupFolder || typeof config.backupFolder !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'backupFolder must be non-empty string');
const error = validateDestPath(config.backupFolder);
if (error) throw error;
rootPath = apiConfig.backupFolder;
rootPath = config.backupFolder;
} else {
if (typeof apiConfig.prefix !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'prefix must be a string');
if (apiConfig.prefix !== '') {
if (path.isAbsolute(apiConfig.prefix)) throw new BoxError(BoxError.BAD_FIELD, 'prefix must be a relative path');
if (path.normalize(apiConfig.prefix) !== apiConfig.prefix) throw new BoxError(BoxError.BAD_FIELD, 'prefix must contain a normalized relative path');
if (typeof config.prefix !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'prefix must be a string');
if (config.prefix !== '') {
if (path.isAbsolute(config.prefix)) throw new BoxError(BoxError.BAD_FIELD, 'prefix must be a relative path');
if (path.normalize(config.prefix) !== config.prefix) throw new BoxError(BoxError.BAD_FIELD, 'prefix must contain a normalized relative path');
}
if (mounts.isManagedProvider(apiConfig.provider)) {
testMountObject = await setupManagedMount(apiConfig, '/mnt/backup-storage-validation'); // this validates mountOptions
rootPath = path.join('/mnt/backup-storage-validation', apiConfig.prefix);
} else if (apiConfig.provider === mounts.MOUNT_TYPE_MOUNTPOINT) {
if (!apiConfig.mountPoint || typeof apiConfig.mountPoint !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'mountPoint must be non-empty string');
const error = validateDestPath(apiConfig.mountPoint);
if (mounts.isManagedProvider(provider)) {
if (!config.mountOptions || typeof config.mountOptions !== 'object') throw new BoxError(BoxError.BAD_FIELD, 'mountOptions must be an object');
const error = mounts.validateMountOptions(provider, config.mountOptions);
if (error) throw error;
const [mountError] = await safe(shell.spawn('mountpoint', ['-q', '--', apiConfig.mountPoint], { timeout: 5000 }));
if (mountError) throw new BoxError(BoxError.BAD_FIELD, `${apiConfig.mountPoint} is not mounted: ${mountError.message}`);
rootPath = path.join(apiConfig.mountPoint, apiConfig.prefix);
testMountObject = await setupManagedMount(provider, config.mountOptions, '/mnt/backup-storage-validation'); // this validates mountOptions
rootPath = path.join('/mnt/backup-storage-validation', config.prefix);
} else if (provider === mounts.MOUNT_TYPE_MOUNTPOINT) {
if (!config.mountPoint || typeof config.mountPoint !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'mountPoint must be non-empty string');
const error = validateDestPath(config.mountPoint);
if (error) throw error;
const [mountError] = await safe(shell.spawn('mountpoint', ['-q', '--', config.mountPoint], { timeout: 5000 }));
if (mountError) throw new BoxError(BoxError.BAD_FIELD, `${config.mountPoint} is not mounted: ${mountError.message}`);
rootPath = path.join(config.mountPoint, config.prefix);
} else {
throw new BoxError(BoxError.BAD_FIELD, `Unknown provider: ${apiConfig.provider}`);
throw new BoxError(BoxError.BAD_FIELD, `Unknown provider: ${provider}`);
}
}
@@ -322,16 +325,25 @@ async function testConfig(apiConfig) {
}
if (testMountObject) await mounts.removeMount('/mnt/backup-storage-validation');
const newConfig = _.pick(config, ['noHardlinks', 'chown', 'preserveAttributes', 'backupFolder', 'prefix', 'mountOptions', 'mountPoint']);
return { provider, id, ...newConfig };
}
function removePrivateFields(apiConfig) {
if (apiConfig.mountOptions && apiConfig.mountOptions.password) apiConfig.mountOptions.password = constants.SECRET_PLACEHOLDER;
if (apiConfig.mountOptions && apiConfig.mountOptions.privateKey) apiConfig.mountOptions.privateKey = constants.SECRET_PLACEHOLDER;
function removePrivateFields(config) {
if (config.mountOptions && config.mountOptions.password) config.mountOptions.password = constants.SECRET_PLACEHOLDER;
if (config.mountOptions && config.mountOptions.privateKey) config.mountOptions.privateKey = constants.SECRET_PLACEHOLDER;
return apiConfig;
delete config.id;
delete config.provider;
return config;
}
function injectPrivateFields(newConfig, currentConfig) {
if (newConfig.mountOptions && currentConfig.mountOptions && newConfig.mountOptions.password === constants.SECRET_PLACEHOLDER) newConfig.mountOptions.password = currentConfig.mountOptions.password;
if (newConfig.mountOptions && currentConfig.mountOptions && newConfig.mountOptions.privateKey === constants.SECRET_PLACEHOLDER) newConfig.mountOptions.privateKey = currentConfig.mountOptions.privateKey;
newConfig.id = currentConfig.id;
newConfig.provider = currentConfig.provider;
}