31df40a841
we changed listDir in c44863a9bb to list
a directory . this broke copy for files since a '/' is added when listing
the file.
420 lines
18 KiB
JavaScript
420 lines
18 KiB
JavaScript
'use strict';
|
|
|
|
exports = module.exports = {
|
|
setup,
|
|
teardown,
|
|
cleanup,
|
|
|
|
verifyConfig,
|
|
removePrivateFields,
|
|
injectPrivateFields,
|
|
|
|
getAvailableSize,
|
|
getStatus,
|
|
|
|
upload,
|
|
download,
|
|
|
|
copy,
|
|
copyDir,
|
|
|
|
exists,
|
|
listDir,
|
|
|
|
remove,
|
|
removeDir,
|
|
};
|
|
|
|
const assert = require('node:assert'),
|
|
BoxError = require('../boxerror.js'),
|
|
constants = require('../constants.js'),
|
|
debug = require('debug')('box:storage/filesystem'),
|
|
df = require('../df.js'),
|
|
fs = require('node:fs'),
|
|
mounts = require('../mounts.js'),
|
|
path = require('node:path'),
|
|
paths = require('../paths.js'),
|
|
safe = require('safetydance'),
|
|
shell = require('../shell.js')('filesystem'),
|
|
_ = require('../underscore.js');
|
|
|
|
|
|
function getRootPath(config) {
|
|
assert.strictEqual(typeof config, 'object');
|
|
|
|
const prefix = config.prefix ?? ''; // can be missing for plain filesystem
|
|
|
|
if (mounts.isManagedProvider(config._provider)) {
|
|
return path.join(config._managedMountPath, prefix);
|
|
} else if (config._provider === mounts.MOUNT_TYPE_MOUNTPOINT) {
|
|
return path.join(config.mountPoint, prefix);
|
|
} else if (config._provider === mounts.MOUNT_TYPE_FILESYSTEM) {
|
|
return path.join(config.backupDir, prefix);
|
|
}
|
|
|
|
throw new BoxError(BoxError.INTERNAL_ERROR, `Unhandled provider: ${config._provider}`);
|
|
}
|
|
|
|
async function getAvailableSize(config) {
|
|
assert.strictEqual(typeof config, 'object');
|
|
|
|
// note that df returns the disk size (as opposed to the apparent size)
|
|
const [error, dfResult] = await safe(df.file(getRootPath(config)));
|
|
if (error) throw new BoxError(BoxError.FS_ERROR, `Error when checking for disk space: ${error.message}`);
|
|
|
|
return dfResult.available;
|
|
}
|
|
|
|
async function getStatus(config) {
|
|
assert.strictEqual(typeof config, 'object');
|
|
|
|
let hostPath;
|
|
if (mounts.isManagedProvider(config._provider)) {
|
|
hostPath = config._managedMountPath;
|
|
} else if (config._provider === 'mountpoint') {
|
|
hostPath = config.mountPoint;
|
|
} else if (config._provider === 'filesystem') {
|
|
hostPath = config.backupDir;
|
|
}
|
|
|
|
return await mounts.getStatus(config._provider, hostPath); // { state, message }
|
|
}
|
|
|
|
function hasChownSupportSync(config) {
|
|
switch (config._provider) {
|
|
case mounts.MOUNT_TYPE_NFS:
|
|
case mounts.MOUNT_TYPE_EXT4:
|
|
case mounts.MOUNT_TYPE_XFS:
|
|
case mounts.MOUNT_TYPE_DISK:
|
|
case mounts.MOUNT_TYPE_FILESYSTEM:
|
|
return true;
|
|
case mounts.MOUNT_TYPE_SSHFS:
|
|
// sshfs can be mounted as root or normal user. when mounted as root, we have to chown since we remove backups as the yellowtent user
|
|
// when mounted as non-root user, files are created as yellowtent user but they are still owned by the non-root user (thus del also works)
|
|
return config.mountOptions.user === 'root';
|
|
case mounts.MOUNT_TYPE_CIFS:
|
|
return true;
|
|
case mounts.MOUNT_TYPE_MOUNTPOINT:
|
|
return config.chown;
|
|
}
|
|
}
|
|
|
|
async function upload(config, remotePath) {
|
|
assert.strictEqual(typeof config, 'object');
|
|
assert.strictEqual(typeof remotePath, 'string');
|
|
|
|
const fullRemotePath = path.join(getRootPath(config), remotePath);
|
|
|
|
const [mkdirError] = await safe(fs.promises.mkdir(path.dirname(fullRemotePath), { recursive: true }));
|
|
if (mkdirError) throw new BoxError(BoxError.FS_ERROR, `Error creating directory ${fullRemotePath}: ${mkdirError.message}`);
|
|
|
|
await safe(fs.promises.unlink(fullRemotePath)); // remove any hardlink
|
|
|
|
return {
|
|
stream: fs.createWriteStream(fullRemotePath, { autoClose: true }),
|
|
async finish() {
|
|
const backupUid = parseInt(process.env.SUDO_UID, 10) || process.getuid(); // in test, upload() may or may not be called via sudo script
|
|
if (hasChownSupportSync(config)) {
|
|
if (!safe.fs.chownSync(fullRemotePath, backupUid, backupUid)) throw new BoxError(BoxError.EXTERNAL_ERROR, `Unable to chown ${fullRemotePath}: ${safe.error.message}`);
|
|
if (!safe.fs.chownSync(path.dirname(fullRemotePath), backupUid, backupUid)) throw new BoxError(BoxError.EXTERNAL_ERROR, `Unable to chown parentdir ${fullRemotePath}: ${safe.error.message}`);
|
|
}
|
|
}
|
|
};
|
|
}
|
|
|
|
async function download(config, remotePath) {
|
|
assert.strictEqual(typeof config, 'object');
|
|
assert.strictEqual(typeof remotePath, 'string');
|
|
|
|
const fullRemotePath = path.join(getRootPath(config), remotePath);
|
|
debug(`download: ${fullRemotePath}`);
|
|
|
|
if (!safe.fs.existsSync(fullRemotePath)) throw new BoxError(BoxError.NOT_FOUND, `File not found: ${fullRemotePath}`);
|
|
|
|
return fs.createReadStream(fullRemotePath);
|
|
}
|
|
|
|
async function exists(config, remotePath) {
|
|
assert.strictEqual(typeof config, 'object');
|
|
assert.strictEqual(typeof remotePath, 'string');
|
|
|
|
const fullRemotePath = path.join(getRootPath(config), remotePath);
|
|
|
|
// do not use existsSync because it does not return EPERM etc
|
|
if (!safe.fs.statSync(fullRemotePath)) {
|
|
if (safe.error && safe.error.code === 'ENOENT') return false;
|
|
if (safe.error) throw new BoxError(BoxError.EXTERNAL_ERROR, `Exists ${fullRemotePath}: ${safe.error.message}`);
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
async function listDir(config, remotePath, batchSize, marker) {
|
|
assert.strictEqual(typeof config, 'object');
|
|
assert.strictEqual(typeof remotePath, 'string');
|
|
assert.strictEqual(typeof batchSize, 'number');
|
|
assert(typeof marker !== 'undefined');
|
|
|
|
const rootPath = getRootPath(config);
|
|
const fullRemotePath = path.join(rootPath, remotePath);
|
|
const stack = marker ? marker.stack : [fullRemotePath];
|
|
const fileStream = marker ? marker.fileStream : [];
|
|
if (!marker) marker = { stack, fileStream };
|
|
|
|
while (stack.length > 0) {
|
|
const currentDir = stack.pop();
|
|
const dirents = await fs.promises.readdir(currentDir, { withFileTypes: true });
|
|
|
|
for (const dirent of dirents) {
|
|
const fullEntryPath = path.join(currentDir, dirent.name);
|
|
|
|
if (dirent.isDirectory()) {
|
|
stack.push(fullEntryPath);
|
|
} else if (dirent.isFile()) { // does not include symlink
|
|
const stat = await fs.promises.lstat(fullEntryPath);
|
|
fileStream.push({ path: path.relative(rootPath, fullEntryPath), size: stat.size });
|
|
}
|
|
}
|
|
|
|
if (fileStream.length >= batchSize) return { entries: fileStream.splice(0, batchSize), marker }; // note: splice also modifies the array
|
|
}
|
|
|
|
if (fileStream.length === 0) return { entries: [], marker: null };
|
|
|
|
return { entries: fileStream.splice(0, batchSize), marker }; // note: splice also modifies the array
|
|
}
|
|
|
|
async function copyInternal(config, fromPath, toPath, options, progressCallback) {
|
|
assert.strictEqual(typeof config, 'object');
|
|
assert.strictEqual(typeof fromPath, 'string');
|
|
assert.strictEqual(typeof toPath, 'string');
|
|
assert.strictEqual(typeof options, 'object');
|
|
assert.strictEqual(typeof progressCallback, 'function');
|
|
|
|
const fullFromPath = path.join(getRootPath(config), fromPath);
|
|
const fullToPath = path.join(getRootPath(config), toPath);
|
|
|
|
const [mkdirError] = await safe(fs.promises.mkdir(path.dirname(fullToPath), { recursive: true }));
|
|
if (mkdirError) throw new BoxError(BoxError.EXTERNAL_ERROR, mkdirError.message);
|
|
|
|
progressCallback({ message: `Copying ${fullFromPath} to ${fullToPath}` });
|
|
|
|
let cpOptions = ((config._provider !== mounts.MOUNT_TYPE_MOUNTPOINT && config._provider !== mounts.MOUNT_TYPE_CIFS) || config.preserveAttributes) ? '-a' : '-d';
|
|
if (options.recursive) cpOptions += 'R';
|
|
cpOptions += config.noHardlinks ? '' : 'l'; // this will hardlink backups saving space
|
|
|
|
if (config._provider === mounts.MOUNT_TYPE_SSHFS) {
|
|
const identityFilePath = path.join(paths.SSHFS_KEYS_DIR, `id_rsa_${config.mountOptions.host}`);
|
|
|
|
const sshOptions = [ '-o', '"StrictHostKeyChecking no"', '-i', identityFilePath, '-p', config.mountOptions.port, `${config.mountOptions.user}@${config.mountOptions.host}` ];
|
|
const sshArgs = sshOptions.concat([ 'cp', cpOptions, path.join(config.prefix, fromPath), path.join(config.prefix, toPath) ]);
|
|
const [remoteCopyError] = await safe(shell.spawn('ssh', sshArgs, { shell: true }));
|
|
if (!remoteCopyError) return;
|
|
if (remoteCopyError.code === 255) throw new BoxError(BoxError.EXTERNAL_ERROR, `SSH connection error: ${remoteCopyError.message}`); // do not attempt fallback copy for ssh errors
|
|
debug('SSH remote copy failed, trying sshfs copy'); // this can happen for sshfs mounted windows server
|
|
}
|
|
|
|
const [copyError] = await safe(shell.spawn('cp', [ cpOptions, fullFromPath, fullToPath ], {}));
|
|
if (copyError) throw new BoxError(BoxError.EXTERNAL_ERROR, copyError.message);
|
|
}
|
|
|
|
async function copy(config, fromPath, toPath, progressCallback) {
|
|
assert.strictEqual(typeof config, 'object');
|
|
assert.strictEqual(typeof fromPath, 'string');
|
|
assert.strictEqual(typeof toPath, 'string');
|
|
assert.strictEqual(typeof progressCallback, 'function');
|
|
|
|
return await copyInternal(config, fromPath, toPath, { recursive: false }, progressCallback);
|
|
}
|
|
|
|
async function copyDir(config, fromPath, toPath, progressCallback) {
|
|
assert.strictEqual(typeof config, 'object');
|
|
assert.strictEqual(typeof fromPath, 'string');
|
|
assert.strictEqual(typeof toPath, 'string');
|
|
assert.strictEqual(typeof progressCallback, 'function');
|
|
|
|
return await copyInternal(config, fromPath, toPath, { recursive: true }, progressCallback);
|
|
}
|
|
|
|
async function remove(config, remotePath) {
|
|
assert.strictEqual(typeof config, 'object');
|
|
assert.strictEqual(typeof remotePath, 'string');
|
|
|
|
const fullRemotePath = path.join(getRootPath(config), remotePath);
|
|
const stat = safe.fs.statSync(fullRemotePath);
|
|
if (!stat) return;
|
|
|
|
if (stat.isFile()) {
|
|
if (!safe.fs.unlinkSync(fullRemotePath)) throw new BoxError(BoxError.EXTERNAL_ERROR, safe.error.message);
|
|
} else if (stat.isDirectory()) {
|
|
if (!safe.fs.rmdirSync(fullRemotePath)) throw new BoxError(BoxError.EXTERNAL_ERROR, safe.error.message);
|
|
}
|
|
}
|
|
|
|
async function removeDir(config, remotePathPrefix, progressCallback) {
|
|
assert.strictEqual(typeof config, 'object');
|
|
assert.strictEqual(typeof remotePathPrefix, 'string');
|
|
assert.strictEqual(typeof progressCallback, 'function');
|
|
|
|
const fullPathPrefix = path.join(getRootPath(config), remotePathPrefix);
|
|
progressCallback({ message: `Removing directory ${fullPathPrefix}` });
|
|
|
|
if (config._provider === mounts.MOUNT_TYPE_SSHFS) {
|
|
const identityFilePath = path.join(paths.SSHFS_KEYS_DIR, `id_rsa_${config.mountOptions.host}`);
|
|
|
|
const sshOptions = [ '-o', '"StrictHostKeyChecking no"', '-i', identityFilePath, '-p', config.mountOptions.port, `${config.mountOptions.user}@${config.mountOptions.host}` ];
|
|
const sshArgs = sshOptions.concat([ 'rm', '-rf', path.join(config.prefix, remotePathPrefix) ]);
|
|
const [remoteRmError] = await safe(shell.spawn('ssh', sshArgs, { shell: true }));
|
|
if (!remoteRmError) return;
|
|
if (remoteRmError.code === 255) throw new BoxError(BoxError.EXTERNAL_ERROR, `SSH connection error: ${remoteRmError.message}`); // do not attempt fallback copy for ssh errors
|
|
debug('SSH remote rm failed, trying sshfs rm'); // this can happen for sshfs mounted windows server
|
|
}
|
|
|
|
const [error] = await safe(shell.spawn('rm', [ '-rf', fullPathPrefix ], {}));
|
|
if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, error.message);
|
|
}
|
|
|
|
function validateDestDir(dir) {
|
|
assert.strictEqual(typeof dir, 'string');
|
|
|
|
if (path.normalize(dir) !== dir) return new BoxError(BoxError.BAD_FIELD, 'backupDir/mountpoint must contain a normalized path');
|
|
if (!path.isAbsolute(dir)) return new BoxError(BoxError.BAD_FIELD, 'backupDir/mountpoint must be an absolute path');
|
|
|
|
if (dir === '/') return new BoxError(BoxError.BAD_FIELD, 'backupDir/mountpoint cannot be /');
|
|
|
|
if (!dir.endsWith('/')) dir = dir + '/'; // ensure trailing slash for the prefix matching to work
|
|
const PROTECTED_PREFIXES = [ '/boot/', '/usr/', '/bin/', '/lib/', '/root/', '/var/lib/', paths.baseDir() ];
|
|
|
|
if (PROTECTED_PREFIXES.some(p => dir.startsWith(p))) return new BoxError(BoxError.BAD_FIELD, `backupDir ${dir} is protected`);
|
|
|
|
return null;
|
|
}
|
|
|
|
async function cleanup(config, progressCallback) {
|
|
assert.strictEqual(typeof config, 'object');
|
|
assert.strictEqual(typeof progressCallback, 'function');
|
|
}
|
|
|
|
function mountObjectFromConfig(config) {
|
|
return {
|
|
description: `Cloudron Managed Mount`,
|
|
hostPath: config._managedMountPath,
|
|
mountType: config._provider,
|
|
mountOptions: config.mountOptions
|
|
};
|
|
}
|
|
|
|
async function setup(config) {
|
|
assert.strictEqual(typeof config, 'object');
|
|
|
|
debug('setup: removing old storage configuration');
|
|
if (!mounts.isManagedProvider(config._provider)) return;
|
|
|
|
const mount = mountObjectFromConfig(config);
|
|
await safe(mounts.removeMount(mount), { debug }); // ignore error
|
|
|
|
debug('setup: setting up new storage configuration');
|
|
await mounts.tryAddMount(mount, { timeout: 10 }); // 10 seconds
|
|
}
|
|
|
|
async function teardown(config) {
|
|
assert.strictEqual(typeof config, 'object');
|
|
|
|
if (!mounts.isManagedProvider(config._provider)) return;
|
|
|
|
await safe(mounts.removeMount(mountObjectFromConfig(config)), { debug }); // ignore error
|
|
}
|
|
|
|
async function verifyConfig({ id, provider, config }) {
|
|
assert.strictEqual(typeof id, 'string');
|
|
assert.strictEqual(typeof provider, 'string');
|
|
assert.strictEqual(typeof config, 'object');
|
|
|
|
if ('noHardlinks' in config && typeof config.noHardlinks !== 'boolean') throw new BoxError(BoxError.BAD_FIELD, 'noHardlinks must be boolean');
|
|
if ('chown' in config && typeof config.chown !== 'boolean') throw new BoxError(BoxError.BAD_FIELD, 'chown must be boolean');
|
|
if ('preserveAttributes' in config && typeof config.preserveAttributes !== 'boolean') throw new BoxError(BoxError.BAD_FIELD, 'preserveAttributes must be boolean');
|
|
|
|
const managedMountPath = path.join(paths.MANAGED_BACKUP_MOUNT_DIR, id);
|
|
|
|
if ('prefix' in config) {
|
|
if (typeof config.prefix !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'prefix must be a string');
|
|
if (config.prefix !== '') {
|
|
if (path.isAbsolute(config.prefix)) throw new BoxError(BoxError.BAD_FIELD, 'prefix must be a relative path');
|
|
if (path.normalize(config.prefix) !== config.prefix) throw new BoxError(BoxError.BAD_FIELD, 'prefix must contain a normalized relative path');
|
|
}
|
|
}
|
|
|
|
if (provider === mounts.MOUNT_TYPE_FILESYSTEM) {
|
|
if (!config.backupDir || typeof config.backupDir !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'backupDir must be non-empty string');
|
|
const error = validateDestDir(config.backupDir);
|
|
if (error) throw error;
|
|
} else {
|
|
if (mounts.isManagedProvider(provider)) {
|
|
if (!config.mountOptions || typeof config.mountOptions !== 'object') throw new BoxError(BoxError.BAD_FIELD, 'mountOptions must be an object');
|
|
|
|
const error = mounts.validateMountOptions(provider, config.mountOptions);
|
|
if (error) throw error;
|
|
|
|
const tmpConfig = {
|
|
description: `Cloudron Validation Mount`,
|
|
hostPath: `${managedMountPath}-validation`,
|
|
mountType: provider,
|
|
mountOptions: config.mountOptions
|
|
};
|
|
|
|
await mounts.tryAddMount(tmpConfig, { timeout: 10 });
|
|
} else if (provider === mounts.MOUNT_TYPE_MOUNTPOINT) {
|
|
if (!config.mountPoint || typeof config.mountPoint !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'mountPoint must be non-empty string');
|
|
const error = validateDestDir(config.mountPoint);
|
|
if (error) throw error;
|
|
|
|
const [mountError] = await safe(shell.spawn('mountpoint', ['-q', '--', config.mountPoint], { timeout: 5000 }));
|
|
if (mountError) throw new BoxError(BoxError.BAD_FIELD, `${config.mountPoint} is not mounted: ${mountError.message}`);
|
|
} else {
|
|
throw new BoxError(BoxError.BAD_FIELD, `Unknown provider: ${provider}`);
|
|
}
|
|
}
|
|
|
|
const newConfig = _.pick(config, ['noHardlinks', 'chown', 'preserveAttributes', 'backupDir', 'prefix', 'mountOptions', 'mountPoint']);
|
|
newConfig._provider = provider;
|
|
|
|
const fullPath = getRootPath({ ...newConfig, _managedMountPath: `${managedMountPath}-validation` });
|
|
if (!safe.fs.mkdirSync(path.join(fullPath, 'snapshot'), { recursive: true }) && safe.error.code !== 'EEXIST') {
|
|
if (safe.error && safe.error.code === 'EACCES') throw new BoxError(BoxError.BAD_FIELD, `Access denied. Create ${fullPath}/snapshot and run "chown yellowtent:yellowtent ${fullPath}" on the server`);
|
|
throw new BoxError(BoxError.BAD_FIELD, safe.error.message);
|
|
}
|
|
|
|
if (!safe.fs.writeFileSync(path.join(fullPath, 'snapshot/cloudron-testfile'), 'testcontent')) {
|
|
throw new BoxError(BoxError.BAD_FIELD, `Unable to create test file as 'yellowtent' user in ${fullPath}: ${safe.error.message}. Check dir/mount permissions`);
|
|
}
|
|
|
|
if (!safe.fs.unlinkSync(path.join(fullPath, 'snapshot/cloudron-testfile'))) {
|
|
throw new BoxError(BoxError.BAD_FIELD, `Unable to remove test file as 'yellowtent' user in ${fullPath}: ${safe.error.message}. Check dir/mount permissions`);
|
|
}
|
|
|
|
if (mounts.isManagedProvider(provider)) {
|
|
await mounts.removeMount({ hostPath: `${managedMountPath}-validation`, mountType: provider, mountOptions: config.mountOptions });
|
|
newConfig._managedMountPath = managedMountPath;
|
|
}
|
|
|
|
return newConfig;
|
|
}
|
|
|
|
function removePrivateFields(config) {
|
|
if (config.mountOptions && config.mountOptions.password) config.mountOptions.password = constants.SECRET_PLACEHOLDER;
|
|
if (config.mountOptions && config.mountOptions.privateKey) config.mountOptions.privateKey = constants.SECRET_PLACEHOLDER;
|
|
|
|
delete config._provider;
|
|
delete config._managedMountPath;
|
|
|
|
return config;
|
|
}
|
|
|
|
function injectPrivateFields(newConfig, currentConfig) {
|
|
if (newConfig.mountOptions && currentConfig.mountOptions && newConfig.mountOptions.password === constants.SECRET_PLACEHOLDER) newConfig.mountOptions.password = currentConfig.mountOptions.password;
|
|
if (newConfig.mountOptions && currentConfig.mountOptions && newConfig.mountOptions.privateKey === constants.SECRET_PLACEHOLDER) newConfig.mountOptions.privateKey = currentConfig.mountOptions.privateKey;
|
|
|
|
newConfig._provider = currentConfig._provider;
|
|
if (currentConfig._managedMountPath) newConfig._managedMountPath = currentConfig._managedMountPath;
|
|
}
|