jjkiers/cloudron-box
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

backups: root ~~canal~~ path surgery

Browse Source 
remove rootPath and getBackupFilePath from the backup target and
make this backend specific.
This commit is contained in:
Girish Ramakrishnan
2025-08-02 01:46:29 +02:00
parent a01e1bad0f
commit c935744f4c
15 changed files with 378 additions and 373 deletions
Download Patch File Download Diff File Expand all files Collapse all files

191
src/storage/filesystem.js
View File

@@ -36,18 +36,35 @@ const assert = require('assert'),
shell = require('../shell.js')('filesystem'),
_ = require('../underscore.js');
function getRootPath(config) {
assert.strictEqual(typeof config, 'object');
const prefix = config.prefix ?? ''; // can be missing for plain filesystem
if (mounts.isManagedProvider(config._provider)) {
return path.join(config._managedMountPath, prefix);
} else if (config._provider === mounts.MOUNT_TYPE_MOUNTPOINT) {
return path.join(config.mountPoint, prefix);
} else if (config._provider === mounts.MOUNT_TYPE_FILESYSTEM) {
return path.join(config.backupFolder, prefix);
}
throw new BoxError(BoxError.INTERNAL_ERROR, `Unhandled provider: ${config._provider}`);
}
async function getAvailableSize(config) {
assert.strictEqual(typeof config, 'object');
// note that df returns the disk size (as opposed to the apparent size)
const [error, dfResult] = await safe(df.file(config.rootPath));
const [error, dfResult] = await safe(df.file(getRootPath(config)));
if (error) throw new BoxError(BoxError.FS_ERROR, `Error when checking for disk space: ${error.message}`);
return dfResult.available;
}
function hasChownSupportSync(config) {
switch (config.provider) {
switch (config._provider) {
case mounts.MOUNT_TYPE_NFS:
case mounts.MOUNT_TYPE_EXT4:
case mounts.MOUNT_TYPE_XFS:
@@ -65,58 +82,64 @@ function hasChownSupportSync(config) {
}
}
async function upload(config, backupFilePath) {
async function upload(config, remotePath) {
assert.strictEqual(typeof config, 'object');
assert.strictEqual(typeof backupFilePath, 'string');
assert.strictEqual(typeof remotePath, 'string');
const [mkdirError] = await safe(fs.promises.mkdir(path.dirname(backupFilePath), { recursive: true }));
if (mkdirError) throw new BoxError(BoxError.FS_ERROR, `Error creating directory ${backupFilePath}: ${mkdirError.message}`);
const fullRemotePath = path.join(getRootPath(config), remotePath);
await safe(fs.promises.unlink(backupFilePath)); // remove any hardlink
const [mkdirError] = await safe(fs.promises.mkdir(path.dirname(fullRemotePath), { recursive: true }));
if (mkdirError) throw new BoxError(BoxError.FS_ERROR, `Error creating directory ${fullRemotePath}: ${mkdirError.message}`);
await safe(fs.promises.unlink(fullRemotePath)); // remove any hardlink
return {
stream: fs.createWriteStream(backupFilePath, { autoClose: true }),
stream: fs.createWriteStream(fullRemotePath, { autoClose: true }),
async finish() {
const backupUid = parseInt(process.env.SUDO_UID, 10) || process.getuid(); // in test, upload() may or may not be called via sudo script
if (hasChownSupportSync(config)) {
if (!safe.fs.chownSync(backupFilePath, backupUid, backupUid)) throw new BoxError(BoxError.EXTERNAL_ERROR, `Unable to chown ${backupFilePath}: ${safe.error.message}`);
if (!safe.fs.chownSync(path.dirname(backupFilePath), backupUid, backupUid)) throw new BoxError(BoxError.EXTERNAL_ERROR, `Unable to chown parentdir ${backupFilePath}: ${safe.error.message}`);
if (!safe.fs.chownSync(fullRemotePath, backupUid, backupUid)) throw new BoxError(BoxError.EXTERNAL_ERROR, `Unable to chown ${fullRemotePath}: ${safe.error.message}`);
if (!safe.fs.chownSync(path.dirname(fullRemotePath), backupUid, backupUid)) throw new BoxError(BoxError.EXTERNAL_ERROR, `Unable to chown parentdir ${fullRemotePath}: ${safe.error.message}`);
}
}
};
}
async function download(config, sourceFilePath) {
async function download(config, remotePath) {
assert.strictEqual(typeof config, 'object');
assert.strictEqual(typeof sourceFilePath, 'string');
assert.strictEqual(typeof remotePath, 'string');
debug(`download: ${sourceFilePath}`);
const fullRemotePath = path.join(getRootPath(config), remotePath);
debug(`download: ${fullRemotePath}`);
if (!safe.fs.existsSync(sourceFilePath)) throw new BoxError(BoxError.NOT_FOUND, `File not found: ${sourceFilePath}`);
if (!safe.fs.existsSync(fullRemotePath)) throw new BoxError(BoxError.NOT_FOUND, `File not found: ${fullRemotePath}`);
return fs.createReadStream(sourceFilePath);
return fs.createReadStream(fullRemotePath);
}
async function exists(config, sourceFilePath) {
async function exists(config, remotePath) {
assert.strictEqual(typeof config, 'object');
assert.strictEqual(typeof sourceFilePath, 'string');
assert.strictEqual(typeof remotePath, 'string');
const fullRemotePath = path.join(getRootPath(config), remotePath);
// do not use existsSync because it does not return EPERM etc
if (!safe.fs.statSync(sourceFilePath)) {
if (!safe.fs.statSync(fullRemotePath)) {
if (safe.error && safe.error.code === 'ENOENT') return false;
if (safe.error) throw new BoxError(BoxError.EXTERNAL_ERROR, `Exists ${sourceFilePath}: ${safe.error.message}`);
if (safe.error) throw new BoxError(BoxError.EXTERNAL_ERROR, `Exists ${fullRemotePath}: ${safe.error.message}`);
}
return true;
}
async function listDir(config, dir, batchSize, marker) {
async function listDir(config, remotePath, batchSize, marker) {
assert.strictEqual(typeof config, 'object');
assert.strictEqual(typeof dir, 'string');
assert.strictEqual(typeof remotePath, 'string');
assert.strictEqual(typeof batchSize, 'number');
assert(typeof marker !== 'undefined');
const stack = marker ? marker.stack : [dir];
const fullRemotePath = path.join(getRootPath(config), remotePath);
const stack = marker ? marker.stack : [fullRemotePath];
const fileStream = marker ? marker.fileStream : [];
if (!marker) marker = { stack, fileStream };
@@ -125,13 +148,13 @@ async function listDir(config, dir, batchSize, marker) {
const dirents = await fs.promises.readdir(currentDir, { withFileTypes: true });
for (const dirent of dirents) {
const fullPath = path.join(currentDir, dirent.name);
const fullEntryPath = path.join(currentDir, dirent.name);
if (dirent.isDirectory()) {
stack.push(fullPath);
stack.push(fullEntryPath);
} else if (dirent.isFile()) { // does not include symlink
const stat = await fs.promises.lstat(fullPath);
fileStream.push({ fullPath, size: stat.size });
const stat = await fs.promises.lstat(fullEntryPath);
fileStream.push({ fullPath: path.relative(fullRemotePath, fullEntryPath), size: stat.size });
}
}
@@ -143,83 +166,88 @@ async function listDir(config, dir, batchSize, marker) {
return { entries: fileStream.splice(0, batchSize), marker }; // note: splice also modifies the array
}
async function copy(config, oldFilePath, newFilePath, progressCallback) {
async function copy(config, fromPath, toPath, progressCallback) {
assert.strictEqual(typeof config, 'object');
assert.strictEqual(typeof oldFilePath, 'string');
assert.strictEqual(typeof newFilePath, 'string');
assert.strictEqual(typeof fromPath, 'string');
assert.strictEqual(typeof toPath, 'string');
assert.strictEqual(typeof progressCallback, 'function');
const [mkdirError] = await safe(fs.promises.mkdir(path.dirname(newFilePath), { recursive: true }));
const fullFromPath = path.join(getRootPath(config), fromPath);
const fullToPath = path.join(getRootPath(config), toPath);
const [mkdirError] = await safe(fs.promises.mkdir(path.dirname(fullToPath), { recursive: true }));
if (mkdirError) throw new BoxError(BoxError.EXTERNAL_ERROR, mkdirError.message);
progressCallback({ message: `Copying ${oldFilePath} to ${newFilePath}` });
progressCallback({ message: `Copying ${fullFromPath} to ${fullToPath}` });
let cpOptions = ((config.provider !== mounts.MOUNT_TYPE_MOUNTPOINT && config.provider !== mounts.MOUNT_TYPE_CIFS) || config.preserveAttributes) ? '-a' : '-dR';
let cpOptions = ((config._provider !== mounts.MOUNT_TYPE_MOUNTPOINT && config._provider !== mounts.MOUNT_TYPE_CIFS) || config.preserveAttributes) ? '-a' : '-dR';
cpOptions += config.noHardlinks ? '' : 'l'; // this will hardlink backups saving space
if (config.provider === mounts.MOUNT_TYPE_SSHFS) {
if (config._provider === mounts.MOUNT_TYPE_SSHFS) {
const identityFilePath = path.join(paths.SSHFS_KEYS_DIR, `id_rsa_${config.mountOptions.host}`);
const sshOptions = [ '-o', '"StrictHostKeyChecking no"', '-i', identityFilePath, '-p', config.mountOptions.port, `${config.mountOptions.user}@${config.mountOptions.host}` ];
const sshArgs = sshOptions.concat([ 'cp', cpOptions, oldFilePath.replace('/mnt/cloudronbackup/', ''), newFilePath.replace('/mnt/cloudronbackup/', '') ]);
const sshArgs = sshOptions.concat([ 'cp', cpOptions, path.join(config.prefix, fromPath), path.join(config.prefix, toPath) ]);
const [remoteCopyError] = await safe(shell.spawn('ssh', sshArgs, { shell: true }));
if (!remoteCopyError) return;
if (remoteCopyError.code === 255) throw new BoxError(BoxError.EXTERNAL_ERROR, `SSH connection error: ${remoteCopyError.message}`); // do not attempt fallback copy for ssh errors
debug('SSH remote copy failed, trying sshfs copy'); // this can happen for sshfs mounted windows server
}
const [copyError] = await safe(shell.spawn('cp', [ cpOptions, oldFilePath, newFilePath ], {}));
const [copyError] = await safe(shell.spawn('cp', [ cpOptions, fullFromPath, fullToPath ], {}));
if (copyError) throw new BoxError(BoxError.EXTERNAL_ERROR, copyError.message);
}
async function remove(config, filename) {
async function remove(config, remotePath) {
assert.strictEqual(typeof config, 'object');
assert.strictEqual(typeof filename, 'string');
assert.strictEqual(typeof remotePath, 'string');
const stat = safe.fs.statSync(filename);
const fullRemotePath = path.join(getRootPath(config), remotePath);
const stat = safe.fs.statSync(fullRemotePath);
if (!stat) return;
if (stat.isFile()) {
if (!safe.fs.unlinkSync(filename)) throw new BoxError(BoxError.EXTERNAL_ERROR, safe.error.message);
if (!safe.fs.unlinkSync(fullRemotePath)) throw new BoxError(BoxError.EXTERNAL_ERROR, safe.error.message);
} else if (stat.isDirectory()) {
if (!safe.fs.rmdirSync(filename, { recursive: false })) throw new BoxError(BoxError.EXTERNAL_ERROR, safe.error.message);
if (!safe.fs.rmdirSync(fullRemotePath, { recursive: false })) throw new BoxError(BoxError.EXTERNAL_ERROR, safe.error.message);
}
}
async function removeDir(config, pathPrefix, progressCallback) {
async function removeDir(config, remotePathPrefix, progressCallback) {
assert.strictEqual(typeof config, 'object');
assert.strictEqual(typeof pathPrefix, 'string');
assert.strictEqual(typeof remotePathPrefix, 'string');
assert.strictEqual(typeof progressCallback, 'function');
progressCallback({ message: `Removing directory ${pathPrefix}` });
const fullPathPrefix = path.join(getRootPath(config), remotePathPrefix);
progressCallback({ message: `Removing directory ${fullPathPrefix}` });
if (config.provider === mounts.MOUNT_TYPE_SSHFS) {
if (config._provider === mounts.MOUNT_TYPE_SSHFS) {
const identityFilePath = path.join(paths.SSHFS_KEYS_DIR, `id_rsa_${config.mountOptions.host}`);
const sshOptions = [ '-o', '"StrictHostKeyChecking no"', '-i', identityFilePath, '-p', config.mountOptions.port, `${config.mountOptions.user}@${config.mountOptions.host}` ];
const sshArgs = sshOptions.concat([ 'rm', '-rf', pathPrefix.replace('/mnt/cloudronbackup/', '') ]);
const sshArgs = sshOptions.concat([ 'rm', '-rf', path.join(config.prefix, remotePathPrefix) ]);
const [remoteRmError] = await safe(shell.spawn('ssh', sshArgs, { shell: true }));
if (!remoteRmError) return;
if (remoteRmError.code === 255) throw new BoxError(BoxError.EXTERNAL_ERROR, `SSH connection error: ${remoteRmError.message}`); // do not attempt fallback copy for ssh errors
debug('SSH remote rm failed, trying sshfs rm'); // this can happen for sshfs mounted windows server
}
const [error] = await safe(shell.spawn('rm', [ '-rf', pathPrefix ], {}));
const [error] = await safe(shell.spawn('rm', [ '-rf', fullPathPrefix ], {}));
if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, error.message);
}
function validateDestPath(folder) {
assert.strictEqual(typeof folder, 'string');
function validateDestDir(dir) {
assert.strictEqual(typeof dir, 'string');
if (path.normalize(folder) !== folder) return new BoxError(BoxError.BAD_FIELD, 'backupFolder/mountpoint must contain a normalized path');
if (!path.isAbsolute(folder)) return new BoxError(BoxError.BAD_FIELD, 'backupFolder/mountpoint must be an absolute path');
if (path.normalize(dir) !== dir) return new BoxError(BoxError.BAD_FIELD, 'backupFolder/mountpoint must contain a normalized path');
if (!path.isAbsolute(dir)) return new BoxError(BoxError.BAD_FIELD, 'backupFolder/mountpoint must be an absolute path');
if (folder === '/') return new BoxError(BoxError.BAD_FIELD, 'backupFolder/mountpoint cannot be /');
if (dir === '/') return new BoxError(BoxError.BAD_FIELD, 'backupFolder/mountpoint cannot be /');
if (!folder.endsWith('/')) folder = folder + '/'; // ensure trailing slash for the prefix matching to work
if (!dir.endsWith('/')) dir = dir + '/'; // ensure trailing slash for the prefix matching to work
const PROTECTED_PREFIXES = [ '/boot/', '/usr/', '/bin/', '/lib/', '/root/', '/var/lib/', paths.baseDir() ];
if (PROTECTED_PREFIXES.some(p => folder.startsWith(p))) return new BoxError(BoxError.BAD_FIELD, 'backupFolder path is protected');
if (PROTECTED_PREFIXES.some(p => dir.startsWith(p))) return new BoxError(BoxError.BAD_FIELD, 'backupFolder path is protected');
return null;
}
@@ -252,19 +280,19 @@ async function setup(config) {
assert.strictEqual(typeof config, 'object');
debug('setup: removing old storage configuration');
if (!mounts.isManagedProvider(config.provider)) return;
if (!mounts.isManagedProvider(config._provider)) return;
const mountPath = path.join(paths.MANAGED_BACKUP_MOUNT_DIR, config.id);
await safe(mounts.removeMount(mountPath), { debug }); // ignore error
debug('setup: setting up new storage configuration');
await setupManagedMount(config.provider, config.mountOptions, mountPath);
await setupManagedMount(config._provider, config.mountOptions, mountPath);
}
async function teardown(config) {
assert.strictEqual(typeof config, 'object');
if (!mounts.isManagedProvider(config.provider)) return;
if (!mounts.isManagedProvider(config._provider)) return;
const mountPath = path.join(paths.MANAGED_BACKUP_MOUNT_DIR, config.id);
await safe(mounts.removeMount(mountPath), { debug }); // ignore error
@@ -279,67 +307,68 @@ async function verifyConfig({ id, provider, config }) {
if ('chown' in config && typeof config.chown !== 'boolean') throw new BoxError(BoxError.BAD_FIELD, 'chown must be boolean');
if ('preserveAttributes' in config && typeof config.preserveAttributes !== 'boolean') throw new BoxError(BoxError.BAD_FIELD, 'preserveAttributes must be boolean');
const managedMountValidationPath = path.join(paths.MANAGED_BACKUP_MOUNT_DIR, `${id}-validation`);
const managedMountPath = path.join(paths.MANAGED_BACKUP_MOUNT_DIR, id);
let rootPath;
if (provider === mounts.MOUNT_TYPE_FILESYSTEM) {
if (!config.backupFolder || typeof config.backupFolder !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'backupFolder must be non-empty string');
const error = validateDestPath(config.backupFolder);
if (error) throw error;
rootPath = config.backupFolder;
} else {
if ('prefix' in config) {
if (typeof config.prefix !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'prefix must be a string');
if (config.prefix !== '') {
if (path.isAbsolute(config.prefix)) throw new BoxError(BoxError.BAD_FIELD, 'prefix must be a relative path');
if (path.normalize(config.prefix) !== config.prefix) throw new BoxError(BoxError.BAD_FIELD, 'prefix must contain a normalized relative path');
}
}
if (provider === mounts.MOUNT_TYPE_FILESYSTEM) {
if (!config.backupFolder || typeof config.backupFolder !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'backupFolder must be non-empty string');
const error = validateDestDir(config.backupFolder);
if (error) throw error;
} else {
if (mounts.isManagedProvider(provider)) {
if (!config.mountOptions || typeof config.mountOptions !== 'object') throw new BoxError(BoxError.BAD_FIELD, 'mountOptions must be an object');
const error = mounts.validateMountOptions(provider, config.mountOptions);
if (error) throw error;
await setupManagedMount(provider, config.mountOptions, managedMountValidationPath);
rootPath = path.join(managedMountValidationPath, config.prefix);
await setupManagedMount(provider, config.mountOptions, `${managedMountPath}-validation`);
} else if (provider === mounts.MOUNT_TYPE_MOUNTPOINT) {
if (!config.mountPoint || typeof config.mountPoint !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'mountPoint must be non-empty string');
const error = validateDestPath(config.mountPoint);
const error = validateDestDir(config.mountPoint);
if (error) throw error;
const [mountError] = await safe(shell.spawn('mountpoint', ['-q', '--', config.mountPoint], { timeout: 5000 }));
if (mountError) throw new BoxError(BoxError.BAD_FIELD, `${config.mountPoint} is not mounted: ${mountError.message}`);
rootPath = path.join(config.mountPoint, config.prefix);
} else {
throw new BoxError(BoxError.BAD_FIELD, `Unknown provider: ${provider}`);
}
}
if (!safe.fs.mkdirSync(path.join(rootPath, 'snapshot'), { recursive: true }) && safe.error.code !== 'EEXIST') {
if (safe.error && safe.error.code === 'EACCES') throw new BoxError(BoxError.BAD_FIELD, `Access denied. Create ${rootPath}/snapshot and run "chown yellowtent:yellowtent ${rootPath}" on the server`);
const tmp = _.pick(config, ['noHardlinks', 'chown', 'preserveAttributes', 'backupFolder', 'prefix', 'mountOptions', 'mountPoint']);
const newConfig = { _provider: provider, _managedMountPath: managedMountPath, ...tmp };
const fullPath = getRootPath(newConfig);
if (!safe.fs.mkdirSync(path.join(fullPath, 'snapshot'), { recursive: true }) && safe.error.code !== 'EEXIST') {
if (safe.error && safe.error.code === 'EACCES') throw new BoxError(BoxError.BAD_FIELD, `Access denied. Create ${fullPath}/snapshot and run "chown yellowtent:yellowtent ${fullPath}" on the server`);
throw new BoxError(BoxError.BAD_FIELD, safe.error.message);
}
if (!safe.fs.writeFileSync(path.join(rootPath, 'cloudron-testfile'), 'testcontent')) {
throw new BoxError(BoxError.BAD_FIELD, `Unable to create test file as 'yellowtent' user in ${rootPath}: ${safe.error.message}. Check dir/mount permissions`);
if (!safe.fs.writeFileSync(path.join(fullPath, 'cloudron-testfile'), 'testcontent')) {
throw new BoxError(BoxError.BAD_FIELD, `Unable to create test file as 'yellowtent' user in ${fullPath}: ${safe.error.message}. Check dir/mount permissions`);
}
if (!safe.fs.unlinkSync(path.join(rootPath, 'cloudron-testfile'))) {
throw new BoxError(BoxError.BAD_FIELD, `Unable to remove test file as 'yellowtent' user in ${rootPath}: ${safe.error.message}. Check dir/mount permissions`);
if (!safe.fs.unlinkSync(path.join(fullPath, 'cloudron-testfile'))) {
throw new BoxError(BoxError.BAD_FIELD, `Unable to remove test file as 'yellowtent' user in ${fullPath}: ${safe.error.message}. Check dir/mount permissions`);
}
if (mounts.isManagedProvider(provider)) await mounts.removeMount(managedMountValidationPath);
if (mounts.isManagedProvider(provider)) await mounts.removeMount(`${managedMountPath}-validation`);
const newConfig = _.pick(config, ['noHardlinks', 'chown', 'preserveAttributes', 'backupFolder', 'prefix', 'mountOptions', 'mountPoint']);
return { provider, id, ...newConfig };
return newConfig;
}
function removePrivateFields(config) {
if (config.mountOptions && config.mountOptions.password) config.mountOptions.password = constants.SECRET_PLACEHOLDER;
if (config.mountOptions && config.mountOptions.privateKey) config.mountOptions.privateKey = constants.SECRET_PLACEHOLDER;
delete config.id;
delete config.provider;
delete config._provider;
delete config._managedMountPath;
return config;
}
@@ -348,6 +377,6 @@ function injectPrivateFields(newConfig, currentConfig) {
if (newConfig.mountOptions && currentConfig.mountOptions && newConfig.mountOptions.password === constants.SECRET_PLACEHOLDER) newConfig.mountOptions.password = currentConfig.mountOptions.password;
if (newConfig.mountOptions && currentConfig.mountOptions && newConfig.mountOptions.privateKey === constants.SECRET_PLACEHOLDER) newConfig.mountOptions.privateKey = currentConfig.mountOptions.privateKey;
newConfig.id = currentConfig.id;
newConfig.provider = currentConfig.provider;
newConfig._provider = currentConfig._provider;
if (currentConfig._managedMountPath) newConfig._managedMountPath = currentConfig._managedMountPath;
}