jjkiers/cloudron-box
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

backups: add setup/teardown

Browse Source 
1. add setup, teardown hooks
2. move the managed mount setup and teardown to filesystem backend
3. remove this vague storage.js

we should convert storageApi into a real object, so we don't have to
keep passing apiConfig around
This commit is contained in:
Girish Ramakrishnan
2025-08-01 14:54:32 +02:00
parent a1a683ec56
commit ea419509f1
14 changed files with 228 additions and 208 deletions
Download Patch File Download Diff File Expand all files Collapse all files

136
src/storage/filesystem.js
View File

@@ -1,6 +1,14 @@
'use strict';
exports = module.exports = {
setup,
teardown,
cleanup,
testConfig,
removePrivateFields,
injectPrivateFields,
getAvailableSize,
upload,
@@ -13,29 +21,15 @@ exports = module.exports = {
remove,
removeDir,
cleanup,
testConfig,
removePrivateFields,
injectPrivateFields
};
const PROVIDER_FILESYSTEM = 'filesystem';
const PROVIDER_MOUNTPOINT = 'mountpoint';
const PROVIDER_SSHFS = 'sshfs';
const PROVIDER_CIFS = 'cifs';
const PROVIDER_XFS = 'xfs';
const PROVIDER_DISK = 'disk'; // replaces xfs and ext4
const PROVIDER_NFS = 'nfs';
const PROVIDER_EXT4 = 'ext4';
const assert = require('assert'),
BoxError = require('../boxerror.js'),
constants = require('../constants.js'),
debug = require('debug')('box:storage/filesystem'),
df = require('../df.js'),
fs = require('fs'),
mounts = require('../mounts.js'),
path = require('path'),
paths = require('../paths.js'),
safe = require('safetydance'),
@@ -53,19 +47,19 @@ async function getAvailableSize(apiConfig) {
function hasChownSupportSync(apiConfig) {
switch (apiConfig.provider) {
case PROVIDER_NFS:
case PROVIDER_EXT4:
case PROVIDER_XFS:
case PROVIDER_DISK:
case PROVIDER_FILESYSTEM:
case mounts.MOUNT_TYPE_NFS:
case mounts.MOUNT_TYPE_EXT4:
case mounts.MOUNT_TYPE_XFS:
case mounts.MOUNT_TYPE_DISK:
case mounts.MOUNT_TYPE_FILESYSTEM:
return true;
case PROVIDER_SSHFS:
case mounts.MOUNT_TYPE_SSHFS:
// sshfs can be mounted as root or normal user. when mounted as root, we have to chown since we remove backups as the yellowtent user
// when mounted as non-root user, files are created as yellowtent user but they are still owned by the non-root user (thus del also works)
return apiConfig.mountOptions.user === 'root';
case PROVIDER_CIFS:
case mounts.MOUNT_TYPE_CIFS:
return true;
case PROVIDER_MOUNTPOINT:
case mounts.MOUNT_TYPE_MOUNTPOINT:
return apiConfig.chown;
}
}
@@ -159,10 +153,10 @@ async function copy(apiConfig, oldFilePath, newFilePath, progressCallback) {
progressCallback({ message: `Copying ${oldFilePath} to ${newFilePath}` });
let cpOptions = ((apiConfig.provider !== PROVIDER_MOUNTPOINT && apiConfig.provider !== PROVIDER_CIFS) || apiConfig.preserveAttributes) ? '-a' : '-dR';
let cpOptions = ((apiConfig.provider !== mounts.MOUNT_TYPE_MOUNTPOINT && apiConfig.provider !== mounts.MOUNT_TYPE_CIFS) || apiConfig.preserveAttributes) ? '-a' : '-dR';
cpOptions += apiConfig.noHardlinks ? '' : 'l'; // this will hardlink backups saving space
if (apiConfig.provider === PROVIDER_SSHFS) {
if (apiConfig.provider === mounts.MOUNT_TYPE_SSHFS) {
const identityFilePath = path.join(paths.SSHFS_KEYS_DIR, `id_rsa_${apiConfig.mountOptions.host}`);
const sshOptions = [ '-o', '"StrictHostKeyChecking no"', '-i', identityFilePath, '-p', apiConfig.mountOptions.port, `${apiConfig.mountOptions.user}@${apiConfig.mountOptions.host}` ];
@@ -198,7 +192,7 @@ async function removeDir(apiConfig, pathPrefix, progressCallback) {
progressCallback({ message: `Removing directory ${pathPrefix}` });
if (apiConfig.provider === PROVIDER_SSHFS) {
if (apiConfig.provider === mounts.MOUNT_TYPE_SSHFS) {
const identityFilePath = path.join(paths.SSHFS_KEYS_DIR, `id_rsa_${apiConfig.mountOptions.host}`);
const sshOptions = [ '-o', '"StrictHostKeyChecking no"', '-i', identityFilePath, '-p', apiConfig.mountOptions.port, `${apiConfig.mountOptions.user}@${apiConfig.mountOptions.host}` ];
@@ -213,7 +207,7 @@ async function removeDir(apiConfig, pathPrefix, progressCallback) {
if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, error.message);
}
function validateBackupTarget(folder) {
function validateDestPath(folder) {
assert.strictEqual(typeof folder, 'string');
if (path.normalize(folder) !== folder) return new BoxError(BoxError.BAD_FIELD, 'backupFolder/mountpoint must contain a normalized path');
@@ -234,6 +228,62 @@ async function cleanup(apiConfig, progressCallback) {
assert.strictEqual(typeof progressCallback, 'function');
}
async function setupManagedMount(apiConfig, hostPath) {
assert.strictEqual(typeof apiConfig, 'object');
assert.strictEqual(typeof hostPath, 'string');
assert(mounts.isManagedProvider(apiConfig.provider));
if (!apiConfig.mountOptions || typeof apiConfig.mountOptions !== 'object') throw new BoxError(BoxError.BAD_FIELD, 'mountOptions must be an object');
const error = mounts.validateMountOptions(apiConfig.provider, apiConfig.mountOptions);
if (error) throw error;
debug(`setupManagedMount: setting up mount at ${hostPath} with ${apiConfig.provider}`);
const newMount = {
name: path.basename(hostPath),
hostPath,
mountType: apiConfig.provider,
mountOptions: apiConfig.mountOptions
};
await mounts.tryAddMount(newMount, { timeout: 10 }); // 10 seconds
return newMount;
}
async function setup(apiConfig) {
assert.strictEqual(typeof apiConfig, 'object');
debug('setup: removing old storage configuration');
if (!mounts.isManagedProvider(apiConfig.provider)) return;
const oldMountObject = {
name: 'backup',
hostPath: paths.MANAGED_BACKUP_MOUNT_DIR,
mountType: apiConfig.provider,
mountOptions: apiConfig.mountOptions // must have already been validated
};
await safe(mounts.removeMount(oldMountObject), { debug }); // ignore error
debug('setup: setting up new storage configuration');
await setupManagedMount(apiConfig, paths.MANAGED_BACKUP_MOUNT_DIR);
}
async function teardown(apiConfig) {
assert.strictEqual(typeof apiConfig, 'object');
if (!mounts.isManagedProvider(apiConfig.provider)) return;
const mountObject = {
name: 'backup',
hostPath: paths.MANAGED_BACKUP_MOUNT_DIR,
mountType: apiConfig.provider,
mountOptions: apiConfig.mountOptions
};
await safe(mounts.removeMount(mountObject), { debug }); // ignore error
}
async function testConfig(apiConfig) {
assert.strictEqual(typeof apiConfig, 'object');
@@ -241,32 +291,32 @@ async function testConfig(apiConfig) {
if ('chown' in apiConfig && typeof apiConfig.chown !== 'boolean') throw new BoxError(BoxError.BAD_FIELD, 'chown must be boolean');
if ('preserveAttributes' in apiConfig && typeof apiConfig.preserveAttributes !== 'boolean') throw new BoxError(BoxError.BAD_FIELD, 'preserveAttributes must be boolean');
let rootPath; // for managed mounts, this uses 'mountPath', which could be some temporary mount location
if (apiConfig.provider === PROVIDER_FILESYSTEM) {
let rootPath, testMountObject;
if (apiConfig.provider === mounts.MOUNT_TYPE_FILESYSTEM) {
if (!apiConfig.backupFolder || typeof apiConfig.backupFolder !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'backupFolder must be non-empty string');
const error = validateBackupTarget(apiConfig.backupFolder);
const error = validateDestPath(apiConfig.backupFolder);
if (error) throw error;
rootPath = apiConfig.backupFolder;
} else { // xfs/cifs/ext4/nfs/mountpoint/sshfs
if (apiConfig.provider === PROVIDER_MOUNTPOINT) {
if (!apiConfig.mountPoint || typeof apiConfig.mountPoint !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'mountPoint must be non-empty string');
const error = validateBackupTarget(apiConfig.mountPoint);
if (error) throw error;
const [mountError] = await safe(shell.spawn('mountpoint', ['-q', '--', apiConfig.mountPoint], { timeout: 5000 }));
if (mountError) throw new BoxError(BoxError.BAD_FIELD, `${apiConfig.mountPoint} is not mounted: ${mountError.message}`);
}
} else {
if (typeof apiConfig.prefix !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'prefix must be a string');
if (apiConfig.prefix !== '') {
if (path.isAbsolute(apiConfig.prefix)) throw new BoxError(BoxError.BAD_FIELD, 'prefix must be a relative path');
if (path.normalize(apiConfig.prefix) !== apiConfig.prefix) throw new BoxError(BoxError.BAD_FIELD, 'prefix must contain a normalized relative path');
}
if (apiConfig.provider === PROVIDER_MOUNTPOINT) {
if (mounts.isManagedProvider(apiConfig.provider)) {
testMountObject = await setupManagedMount(apiConfig, '/mnt/backup-storage-validation'); // this validates mountOptions
rootPath = path.join('/mnt/backup-storage-validation', apiConfig.prefix);
} else if (apiConfig.provider === mounts.MOUNT_TYPE_MOUNTPOINT) {
if (!apiConfig.mountPoint || typeof apiConfig.mountPoint !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'mountPoint must be non-empty string');
const error = validateDestPath(apiConfig.mountPoint);
if (error) throw error;
const [mountError] = await safe(shell.spawn('mountpoint', ['-q', '--', apiConfig.mountPoint], { timeout: 5000 }));
if (mountError) throw new BoxError(BoxError.BAD_FIELD, `${apiConfig.mountPoint} is not mounted: ${mountError.message}`);
rootPath = path.join(apiConfig.mountPoint, apiConfig.prefix);
} else {
rootPath = path.join(apiConfig.mountPath, apiConfig.prefix);
throw new BoxError(BoxError.BAD_FIELD, `Unknown provider: ${apiConfig.provider}`);
}
}
@@ -282,6 +332,8 @@ async function testConfig(apiConfig) {
if (!safe.fs.unlinkSync(path.join(rootPath, 'cloudron-testfile'))) {
throw new BoxError(BoxError.BAD_FIELD, `Unable to remove test file as 'yellowtent' user in ${rootPath}: ${safe.error.message}. Check dir/mount permissions`);
}
if (testMountObject) await mounts.removeMount(testMountObject);
}
function removePrivateFields(apiConfig) {