'use strict'; exports = module.exports = { getAvailableSize, upload, download, copy, exists, listDir, remove, removeDir, testConfig, removePrivateFields, injectPrivateFields }; const PROVIDER_FILESYSTEM = 'filesystem'; const PROVIDER_MOUNTPOINT = 'mountpoint'; const PROVIDER_SSHFS = 'sshfs'; const PROVIDER_CIFS = 'cifs'; const PROVIDER_XFS = 'xfs'; const PROVIDER_DISK = 'disk'; // replaces xfs and ext4 const PROVIDER_NFS = 'nfs'; const PROVIDER_EXT4 = 'ext4'; const assert = require('assert'), BoxError = require('../boxerror.js'), constants = require('../constants.js'), debug = require('debug')('box:storage/filesystem'), df = require('../df.js'), fs = require('fs'), path = require('path'), paths = require('../paths.js'), readdirp = require('readdirp'), safe = require('safetydance'), shell = require('../shell.js')('filesystem'); async function getAvailableSize(apiConfig) { assert.strictEqual(typeof apiConfig, 'object'); // note that df returns the disk size (as opposed to the apparent size) const [error, dfResult] = await safe(df.file(apiConfig.rootPath)); if (error) throw new BoxError(BoxError.FS_ERROR, `Error when checking for disk space: ${error.message}`); return dfResult.available; } function hasChownSupportSync(apiConfig) { switch (apiConfig.provider) { case PROVIDER_NFS: case PROVIDER_EXT4: case PROVIDER_XFS: case PROVIDER_DISK: case PROVIDER_FILESYSTEM: return true; case PROVIDER_SSHFS: // sshfs can be mounted as root or normal user. when mounted as root, we have to chown since we remove backups as the yellowtent user // when mounted as non-root user, files are created as yellowtent user but they are still owned by the non-root user (thus del also works) return apiConfig.mountOptions.user === 'root'; case PROVIDER_CIFS: return true; case PROVIDER_MOUNTPOINT: return apiConfig.chown; } } async function upload(apiConfig, backupFilePath) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof backupFilePath, 'string'); const [mkdirError] = await safe(fs.promises.mkdir(path.dirname(backupFilePath), { recursive: true })); if (mkdirError) throw new BoxError(BoxError.FS_ERROR, `Error creating directory ${backupFilePath}: ${mkdirError.message}`); await safe(fs.promises.unlink(backupFilePath)); // remove any hardlink return { stream: fs.createWriteStream(backupFilePath, { autoClose: true }), async finish() { const backupUid = parseInt(process.env.SUDO_UID, 10) || process.getuid(); // in test, upload() may or may not be called via sudo script if (hasChownSupportSync(apiConfig)) { if (!safe.fs.chownSync(backupFilePath, backupUid, backupUid)) throw new BoxError(BoxError.EXTERNAL_ERROR, `Unable to chown ${backupFilePath}: ${safe.error.message}`); if (!safe.fs.chownSync(path.dirname(backupFilePath), backupUid, backupUid)) throw new BoxError(BoxError.EXTERNAL_ERROR, `Unable to chown parentdir ${backupFilePath}: ${safe.error.message}`); } } }; } async function download(apiConfig, sourceFilePath) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof sourceFilePath, 'string'); debug(`download: ${sourceFilePath}`); if (!safe.fs.existsSync(sourceFilePath)) throw new BoxError(BoxError.NOT_FOUND, `File not found: ${sourceFilePath}`); return fs.createReadStream(sourceFilePath); } async function exists(apiConfig, sourceFilePath) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof sourceFilePath, 'string'); // do not use existsSync because it does not return EPERM etc if (!safe.fs.statSync(sourceFilePath)) { if (safe.error && safe.error.code === 'ENOENT') return false; if (safe.error) throw new BoxError(BoxError.EXTERNAL_ERROR, `Exists ${sourceFilePath}: ${safe.error.message}`); } return true; } function listDir(apiConfig, dir, batchSize, iteratorCallback, callback) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof dir, 'string'); assert.strictEqual(typeof batchSize, 'number'); assert.strictEqual(typeof iteratorCallback, 'function'); assert.strictEqual(typeof callback, 'function'); var entries = []; var entryStream = readdirp(dir, { type: 'files', alwaysStat: true, lstat: true }); entryStream.on('data', function (entryInfo) { if (entryInfo.stats.isSymbolicLink()) return; entries.push({ fullPath: entryInfo.fullPath }); if (entries.length < batchSize) return; entryStream.pause(); iteratorCallback(entries, function (error) { if (error) return callback(error); entries = []; entryStream.resume(); }); }); entryStream.on('warn', function (error) { debug('listDir: warning. %o', error); }); entryStream.on('end', function () { iteratorCallback(entries, callback); }); } async function copy(apiConfig, oldFilePath, newFilePath, progressCallback) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof oldFilePath, 'string'); assert.strictEqual(typeof newFilePath, 'string'); assert.strictEqual(typeof progressCallback, 'function'); const [mkdirError] = await safe(fs.promises.mkdir(path.dirname(newFilePath), { recursive: true })); if (mkdirError) throw new BoxError(BoxError.EXTERNAL_ERROR, mkdirError.message); progressCallback({ message: `Copying ${oldFilePath} to ${newFilePath}` }); let cpOptions = ((apiConfig.provider !== PROVIDER_MOUNTPOINT && apiConfig.provider !== PROVIDER_CIFS) || apiConfig.preserveAttributes) ? '-a' : '-dR'; cpOptions += apiConfig.noHardlinks ? '' : 'l'; // this will hardlink backups saving space if (apiConfig.provider === PROVIDER_SSHFS) { const identityFilePath = path.join(paths.SSHFS_KEYS_DIR, `id_rsa_${apiConfig.mountOptions.host}`); const sshOptions = [ '-o', '"StrictHostKeyChecking no"', '-i', identityFilePath, '-p', apiConfig.mountOptions.port, `${apiConfig.mountOptions.user}@${apiConfig.mountOptions.host}` ]; const sshArgs = sshOptions.concat([ 'cp', cpOptions, oldFilePath.replace('/mnt/cloudronbackup/', ''), newFilePath.replace('/mnt/cloudronbackup/', '') ]); const [remoteCopyError] = await safe(shell.spawn('ssh', sshArgs, { shell: true })); if (!remoteCopyError) return; if (remoteCopyError.code === 255) throw new BoxError(BoxError.EXTERNAL_ERROR, `SSH connection error: ${remoteCopyError.message}`); // do not attempt fallback copy for ssh errors debug('SSH remote copy failed, trying ssfs copy'); // this can happen for sshfs mounted windows server } const [copyError] = await safe(shell.spawn('cp', [ cpOptions, oldFilePath, newFilePath ], {})); if (copyError) throw new BoxError(BoxError.EXTERNAL_ERROR, copyError.message); } async function remove(apiConfig, filename) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof filename, 'string'); const stat = safe.fs.statSync(filename); if (!stat) return; if (stat.isFile()) { if (!safe.fs.unlinkSync(filename)) throw new BoxError(BoxError.EXTERNAL_ERROR, safe.error.message); } else if (stat.isDirectory()) { if (!safe.fs.rmdirSync(filename, { recursive: false })) throw new BoxError(BoxError.EXTERNAL_ERROR, safe.error.message); } } async function removeDir(apiConfig, pathPrefix, progressCallback) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof pathPrefix, 'string'); assert.strictEqual(typeof progressCallback, 'function'); progressCallback({ message: `Removing directory ${pathPrefix}` }); const [error] = await safe(shell.spawn('rm', [ '-rf', pathPrefix ], {})); if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, error.message); } function validateBackupTarget(folder) { assert.strictEqual(typeof folder, 'string'); if (path.normalize(folder) !== folder) return new BoxError(BoxError.BAD_FIELD, 'backupFolder/mountpoint must contain a normalized path'); if (!path.isAbsolute(folder)) return new BoxError(BoxError.BAD_FIELD, 'backupFolder/mountpoint must be an absolute path'); if (folder === '/') return new BoxError(BoxError.BAD_FIELD, 'backupFolder/mountpoint cannot be /'); if (!folder.endsWith('/')) folder = folder + '/'; // ensure trailing slash for the prefix matching to work const PROTECTED_PREFIXES = [ '/boot/', '/usr/', '/bin/', '/lib/', '/root/', '/var/lib/', paths.baseDir() ]; if (PROTECTED_PREFIXES.some(p => folder.startsWith(p))) return new BoxError(BoxError.BAD_FIELD, 'backupFolder path is protected'); return null; } async function testConfig(apiConfig) { assert.strictEqual(typeof apiConfig, 'object'); if ('noHardlinks' in apiConfig && typeof apiConfig.noHardlinks !== 'boolean') throw new BoxError(BoxError.BAD_FIELD, 'noHardlinks must be boolean'); if ('chown' in apiConfig && typeof apiConfig.chown !== 'boolean') throw new BoxError(BoxError.BAD_FIELD, 'chown must be boolean'); if ('preserveAttributes' in apiConfig && typeof apiConfig.preserveAttributes !== 'boolean') throw new BoxError(BoxError.BAD_FIELD, 'preserveAttributes must be boolean'); let rootPath; // for managed mounts, this uses 'mountPath', which could be some temporary mount location if (apiConfig.provider === PROVIDER_FILESYSTEM) { if (!apiConfig.backupFolder || typeof apiConfig.backupFolder !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'backupFolder must be non-empty string'); const error = validateBackupTarget(apiConfig.backupFolder); if (error) throw error; rootPath = apiConfig.backupFolder; } else { // xfs/cifs/ext4/nfs/mountpoint/sshfs if (apiConfig.provider === PROVIDER_MOUNTPOINT) { if (!apiConfig.mountPoint || typeof apiConfig.mountPoint !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'mountPoint must be non-empty string'); const error = validateBackupTarget(apiConfig.mountPoint); if (error) throw error; const [mountError] = await safe(shell.spawn('mountpoint', ['-q', '--', apiConfig.mountPoint], { timeout: 5000 })); if (mountError) throw new BoxError(BoxError.BAD_FIELD, `${apiConfig.mountPoint} is not mounted: ${mountError.message}`); } if (typeof apiConfig.prefix !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'prefix must be a string'); if (apiConfig.prefix !== '') { if (path.isAbsolute(apiConfig.prefix)) throw new BoxError(BoxError.BAD_FIELD, 'prefix must be a relative path'); if (path.normalize(apiConfig.prefix) !== apiConfig.prefix) throw new BoxError(BoxError.BAD_FIELD, 'prefix must contain a normalized relative path'); } if (apiConfig.provider === PROVIDER_MOUNTPOINT) { rootPath = path.join(apiConfig.mountPoint, apiConfig.prefix); } else { rootPath = path.join(apiConfig.mountPath, apiConfig.prefix); } } if (!safe.fs.mkdirSync(path.join(rootPath, 'snapshot'), { recursive: true }) && safe.error.code !== 'EEXIST') { if (safe.error && safe.error.code === 'EACCES') throw new BoxError(BoxError.BAD_FIELD, `Access denied. Create ${rootPath}/snapshot and run "chown yellowtent:yellowtent ${rootPath}" on the server`); throw new BoxError(BoxError.BAD_FIELD, safe.error.message); } if (!safe.fs.writeFileSync(path.join(rootPath, 'cloudron-testfile'), 'testcontent')) { throw new BoxError(BoxError.BAD_FIELD, `Unable to create test file as 'yellowtent' user in ${rootPath}: ${safe.error.message}. Check dir/mount permissions`); } if (!safe.fs.unlinkSync(path.join(rootPath, 'cloudron-testfile'))) { throw new BoxError(BoxError.BAD_FIELD, `Unable to remove test file as 'yellowtent' user in ${rootPath}: ${safe.error.message}. Check dir/mount permissions`); } } function removePrivateFields(apiConfig) { if (apiConfig.mountOptions && apiConfig.mountOptions.password) apiConfig.mountOptions.password = constants.SECRET_PLACEHOLDER; if (apiConfig.mountOptions && apiConfig.mountOptions.privateKey) apiConfig.mountOptions.privateKey = constants.SECRET_PLACEHOLDER; return apiConfig; } function injectPrivateFields(newConfig, currentConfig) { if (newConfig.mountOptions && currentConfig.mountOptions && newConfig.mountOptions.password === constants.SECRET_PLACEHOLDER) newConfig.mountOptions.password = currentConfig.mountOptions.password; if (newConfig.mountOptions && currentConfig.mountOptions && newConfig.mountOptions.privateKey === constants.SECRET_PLACEHOLDER) newConfig.mountOptions.privateKey = currentConfig.mountOptions.privateKey; }