'use strict'; exports = module.exports = { getRootPath, checkPreconditions, upload, download, copy, exists, listDir, remove, removeDir, remount, testConfig, removePrivateFields, injectPrivateFields }; const PROVIDER_FILESYSTEM = 'filesystem'; const PROVIDER_MOUNTPOINT = 'mountpoint'; const PROVIDER_SSHFS = 'sshfs'; const PROVIDER_CIFS = 'cifs'; const PROVIDER_NFS = 'nfs'; const PROVIDER_EXT4 = 'ext4'; const assert = require('assert'), BoxError = require('../boxerror.js'), constants = require('../constants.js'), DataLayout = require('../datalayout.js'), debug = require('debug')('box:storage/filesystem'), df = require('@sindresorhus/df'), EventEmitter = require('events'), fs = require('fs'), mounts = require('../mounts.js'), path = require('path'), paths = require('../paths.js'), prettyBytes = require('pretty-bytes'), readdirp = require('readdirp'), safe = require('safetydance'), shell = require('../shell.js'); // storage api function getRootPath(apiConfig) { assert.strictEqual(typeof apiConfig, 'object'); switch (apiConfig.provider) { case PROVIDER_SSHFS: case PROVIDER_NFS: case PROVIDER_CIFS: case PROVIDER_EXT4: return path.join(paths.MANAGED_BACKUP_MOUNT_DIR, apiConfig.prefix); case PROVIDER_MOUNTPOINT: return path.join(apiConfig.mountPoint, apiConfig.prefix); case PROVIDER_FILESYSTEM: return apiConfig.backupFolder; } } // the du call in the function below requires root async function checkPreconditions(apiConfig, dataLayout) { assert.strictEqual(typeof apiConfig, 'object'); assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout'); let used = 0; for (let localPath of dataLayout.localPaths()) { debug(`checkPreconditions: getting disk usage of ${localPath}`); let result = safe.child_process.execSync(`du -Dsb ${localPath}`, { encoding: 'utf8' }); if (!result) throw new BoxError(BoxError.FS_ERROR, `du error: ${safe.error.message}`); used += parseInt(result, 10); } debug(`checkPreconditions: ${used} bytes`); const [error, result] = await safe(df.file(getRootPath(apiConfig))); if (error) throw new BoxError(BoxError.FS_ERROR, `Error when checking for disk space: ${error.message}`); // Check filesystem is mounted so we don't write into the actual folder on disk if (apiConfig.provider === PROVIDER_SSHFS || apiConfig.provider === PROVIDER_CIFS || apiConfig.provider === PROVIDER_NFS || apiConfig.provider === PROVIDER_EXT4) { if (result.mountpoint !== paths.MANAGED_BACKUP_MOUNT_DIR) throw new BoxError(BoxError.FS_ERROR, 'Backup target is not mounted'); } else if (apiConfig.provider === PROVIDER_MOUNTPOINT) { if (result.mountpoint === '/') throw new BoxError(BoxError.FS_ERROR, `${apiConfig.backupFolder} is not mounted`); } const needed = 0.6 * used + (1024 * 1024 * 1024); // check if there is atleast 1GB left afterwards. aim for 60% because rsync/tgz won't need full 100% if (result.available <= needed) throw new BoxError(BoxError.FS_ERROR, `Not enough disk space for backup. Needed: ${prettyBytes(needed)} Available: ${prettyBytes(result.available)}`); } function hasChownSupportSync(apiConfig) { switch (apiConfig.provider) { case PROVIDER_NFS: case PROVIDER_EXT4: case PROVIDER_FILESYSTEM: return true; case PROVIDER_SSHFS: // sshfs can be mounted as root or normal user. when mounted as root, we have to chown since we remove backups as the yellowtent user // when mounted as non-root user, files are created as yellowtent user but they are still owned by the non-root user (thus del also works) return apiConfig.mountOptions.user === 'root'; case PROVIDER_CIFS: return true; case PROVIDER_MOUNTPOINT: return apiConfig.chown; } } function upload(apiConfig, backupFilePath, sourceStream, callback) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof backupFilePath, 'string'); assert.strictEqual(typeof sourceStream, 'object'); assert.strictEqual(typeof callback, 'function'); fs.mkdir(path.dirname(backupFilePath), { recursive: true }, function (error) { if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, error.message)); safe.fs.unlinkSync(backupFilePath); // remove any hardlink var fileStream = fs.createWriteStream(backupFilePath); // this pattern is required to ensure that the file got created before 'finish' fileStream.on('open', function () { sourceStream.pipe(fileStream); }); fileStream.on('error', function (error) { debug('[%s] upload: out stream error.', backupFilePath, error); callback(new BoxError(BoxError.EXTERNAL_ERROR, error.message)); }); fileStream.on('finish', function () { const backupUid = parseInt(process.env.SUDO_UID, 10) || process.getuid(); // in test, upload() may or may not be called via sudo script if (hasChownSupportSync(apiConfig)) { if (!safe.fs.chownSync(backupFilePath, backupUid, backupUid)) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Unable to chown:' + safe.error.message)); if (!safe.fs.chownSync(path.dirname(backupFilePath), backupUid, backupUid)) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Unable to chown:' + safe.error.message)); } debug('upload %s: done.', backupFilePath); callback(null); }); }); } function download(apiConfig, sourceFilePath, callback) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof sourceFilePath, 'string'); assert.strictEqual(typeof callback, 'function'); debug(`download: ${sourceFilePath}`); if (!safe.fs.existsSync(sourceFilePath)) return callback(new BoxError(BoxError.NOT_FOUND, `File not found: ${sourceFilePath}`)); var fileStream = fs.createReadStream(sourceFilePath); callback(null, fileStream); } async function exists(apiConfig, sourceFilePath) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof sourceFilePath, 'string'); // do not use existsSync because it does not return EPERM etc if (!safe.fs.statSync(sourceFilePath)) { if (safe.error && safe.error.code === 'ENOENT') return false; if (safe.error) throw new BoxError(BoxError.EXTERNAL_ERROR, `Exists ${sourceFilePath}: ${safe.error.message}`); } return true; } function listDir(apiConfig, dir, batchSize, iteratorCallback, callback) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof dir, 'string'); assert.strictEqual(typeof batchSize, 'number'); assert.strictEqual(typeof iteratorCallback, 'function'); assert.strictEqual(typeof callback, 'function'); var entries = []; var entryStream = readdirp(dir, { type: 'files', alwaysStat: true, lstat: true }); entryStream.on('data', function (entryInfo) { if (entryInfo.stats.isSymbolicLink()) return; entries.push({ fullPath: entryInfo.fullPath }); if (entries.length < batchSize) return; entryStream.pause(); iteratorCallback(entries, function (error) { if (error) return callback(error); entries = []; entryStream.resume(); }); }); entryStream.on('warn', function (error) { debug('listDir: warning ', error); }); entryStream.on('end', function () { iteratorCallback(entries, callback); }); } function copy(apiConfig, oldFilePath, newFilePath) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof oldFilePath, 'string'); assert.strictEqual(typeof newFilePath, 'string'); var events = new EventEmitter(); fs.mkdir(path.dirname(newFilePath), { recursive: true }, function (error) { if (error) return events.emit('done', new BoxError(BoxError.EXTERNAL_ERROR, error.message)); events.emit('progress', `Copying ${oldFilePath} to ${newFilePath}`); let cpOptions = ((apiConfig.provider !== PROVIDER_MOUNTPOINT && apiConfig.provider !== PROVIDER_CIFS) || apiConfig.preserveAttributes) ? '-a' : '-dR'; cpOptions += apiConfig.noHardlinks ? '' : 'l'; // this will hardlink backups saving space shell.spawn('copy', '/bin/cp', [ cpOptions, oldFilePath, newFilePath ], { }, function (error) { if (error) return events.emit('done', new BoxError(BoxError.EXTERNAL_ERROR, error.message)); events.emit('done', null); }); }); return events; } async function remove(apiConfig, filename) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof filename, 'string'); const stat = safe.fs.statSync(filename); if (!stat) return; if (stat.isFile()) { if (!safe.fs.unlinkSync(filename)) throw new BoxError(BoxError.EXTERNAL_ERROR, safe.error.message); } else if (stat.isDirectory()) { if (!safe.fs.rmSync(filename, { recursive: true })) throw new BoxError(BoxError.EXTERNAL_ERROR, safe.error.message); } } async function removeDir(apiConfig, pathPrefix, progressCallback) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof pathPrefix, 'string'); assert.strictEqual(typeof progressCallback, 'function'); progressCallback({ message: `Removing directory ${pathPrefix}` }); const [error] = await safe(shell.promises.spawn('removeDir', '/bin/rm', [ '-rf', pathPrefix ], { })); if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, error.message); } function validateBackupTarget(folder) { assert.strictEqual(typeof folder, 'string'); if (path.normalize(folder) !== folder) return new BoxError(BoxError.BAD_FIELD, 'backupFolder/mountpoint must contain a normalized path'); if (!path.isAbsolute(folder)) return new BoxError(BoxError.BAD_FIELD, 'backupFolder/mountpoint must be an absolute path'); if (folder === '/') return new BoxError(BoxError.BAD_FIELD, 'backupFolder/mountpoint cannot be /'); if (!folder.endsWith('/')) folder = folder + '/'; // ensure trailing slash for the prefix matching to work const PROTECTED_PREFIXES = [ '/boot/', '/usr/', '/bin/', '/lib/', '/root/', '/var/lib/', paths.baseDir() ]; if (PROTECTED_PREFIXES.some(p => folder.startsWith(p))) return new BoxError(BoxError.BAD_FIELD, 'backupFolder path is protected'); return null; } async function remount(apiConfig) { assert.strictEqual(typeof apiConfig, 'object'); if (apiConfig.provider === PROVIDER_FILESYSTEM || apiConfig.provider === PROVIDER_MOUNTPOINT) return; await mounts.remount(mounts.mountObjectFromBackupConfig(apiConfig)); } async function testConfig(apiConfig) { assert.strictEqual(typeof apiConfig, 'object'); if ('noHardlinks' in apiConfig && typeof apiConfig.noHardlinks !== 'boolean') throw new BoxError(BoxError.BAD_FIELD, 'noHardlinks must be boolean'); if ('chown' in apiConfig && typeof apiConfig.chown !== 'boolean') throw new BoxError(BoxError.BAD_FIELD, 'chown must be boolean'); if (apiConfig.provider === PROVIDER_FILESYSTEM) { if (!apiConfig.backupFolder || typeof apiConfig.backupFolder !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'backupFolder must be non-empty string'); const error = validateBackupTarget(apiConfig.backupFolder); if (error) throw error; } else { // cifs/ext4/nfs/mountpoint/sshfs if (apiConfig.provider === PROVIDER_MOUNTPOINT) { if (!apiConfig.mountPoint || typeof apiConfig.mountPoint !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'mountPoint must be non-empty string'); const error = validateBackupTarget(apiConfig.mountPoint); if (error) throw error; } if (typeof apiConfig.prefix !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'prefix must be a string'); if (apiConfig.prefix !== '') { if (path.isAbsolute(apiConfig.prefix)) throw new BoxError(BoxError.BAD_FIELD, 'prefix must be a relative path'); if (path.normalize(apiConfig.prefix) !== apiConfig.prefix) throw new BoxError(BoxError.BAD_FIELD, 'prefix must contain a normalized relative path'); } } if (apiConfig.provider === PROVIDER_MOUNTPOINT) { if (!safe.child_process.execSync(`mountpoint -q -- ${apiConfig.mountPoint}`)) throw new BoxError(BoxError.BAD_FIELD, `${apiConfig.mountPoint} is not mounted`); } const basePath = getRootPath(apiConfig); const field = apiConfig.provider === PROVIDER_FILESYSTEM ? 'backupFolder' : 'mountPoint'; if (!safe.fs.mkdirSync(path.join(basePath, 'snapshot'), { recursive: true }) && safe.error.code !== 'EEXIST') { if (safe.error && safe.error.code === 'EACCES') throw new BoxError(BoxError.BAD_FIELD, `Access denied. Create the directory and run "chown yellowtent:yellowtent ${basePath}" on the server`, { field }); throw new BoxError(BoxError.BAD_FIELD, safe.error.message, { field }); } if (!safe.fs.writeFileSync(path.join(basePath, 'cloudron-testfile'), 'testcontent')) { throw new BoxError(BoxError.BAD_FIELD, `Unable to create test file as 'yellowtent' user in ${basePath}: ${safe.error.message}. Check dir/mount permissions`, { field }); } if (!safe.fs.unlinkSync(path.join(basePath, 'cloudron-testfile'))) { throw new BoxError(BoxError.BAD_FIELD, `Unable to remove test file as 'yellowtent' user in ${basePath}: ${safe.error.message}. Check dir/mount permissions`, { field }); } } function removePrivateFields(apiConfig) { if (apiConfig.mountOptions && apiConfig.mountOptions.password) apiConfig.mountOptions.password = constants.SECRET_PLACEHOLDER; if (apiConfig.mountOptions && apiConfig.mountOptions.privateKey) apiConfig.mountOptions.privateKey = constants.SECRET_PLACEHOLDER; return apiConfig; } function injectPrivateFields(newConfig, currentConfig) { if (newConfig.mountOptions && currentConfig.mountOptions && newConfig.mountOptions.password === constants.SECRET_PLACEHOLDER) newConfig.mountOptions.password = currentConfig.mountOptions.password; if (newConfig.mountOptions && currentConfig.mountOptions && newConfig.mountOptions.privateKey === constants.SECRET_PLACEHOLDER) newConfig.mountOptions.privateKey = currentConfig.mountOptions.privateKey; }