'use strict'; exports = module.exports = { getBackupPath: getBackupPath, checkPreconditions: checkPreconditions, upload: upload, download: download, copy: copy, listDir: listDir, remove: remove, removeDir: removeDir, testConfig: testConfig, removePrivateFields: removePrivateFields, injectPrivateFields: injectPrivateFields }; const PROVIDER_FILESYSTEM = 'filesystem'; const PROVIDER_SSHFS = 'sshfs'; const PROVIDER_CIFS = 'cifs'; const PROVIDER_NFS = 'nfs'; var assert = require('assert'), BoxError = require('../boxerror.js'), DataLayout = require('../datalayout.js'), debug = require('debug')('box:storage/filesystem'), df = require('@sindresorhus/df'), EventEmitter = require('events'), fs = require('fs'), path = require('path'), paths = require('../paths.js'), prettyBytes = require('pretty-bytes'), readdirp = require('readdirp'), safe = require('safetydance'), shell = require('../shell.js'); // storage api function getBackupPath(apiConfig) { assert.strictEqual(typeof apiConfig, 'object'); if (apiConfig.provider === PROVIDER_SSHFS) return path.join(apiConfig.mountPoint, apiConfig.prefix); if (apiConfig.provider === PROVIDER_CIFS) return path.join(apiConfig.mountPoint, apiConfig.prefix); if (apiConfig.provider === PROVIDER_NFS) return path.join(apiConfig.mountPoint, apiConfig.prefix); return apiConfig.backupFolder; } // the du call in the function below requires root function checkPreconditions(apiConfig, dataLayout, callback) { assert.strictEqual(typeof apiConfig, 'object'); assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout'); assert.strictEqual(typeof callback, 'function'); let used = 0; for (let localPath of dataLayout.localPaths()) { debug(`checkPreconditions: getting disk usage of ${localPath}`); let result = safe.child_process.execSync(`du -Dsb ${localPath}`, { encoding: 'utf8' }); if (!result) return callback(new BoxError(BoxError.FS_ERROR, safe.error)); used += parseInt(result, 10); } debug(`checkPreconditions: ${used} bytes`); df.file(getBackupPath(apiConfig)).then(function (result) { // Check filesystem is mounted so we don't write into the actual folder on disk if (apiConfig.provider === PROVIDER_SSHFS || apiConfig.provider === PROVIDER_CIFS || apiConfig.provider === PROVIDER_NFS) { if (result.mountpoint !== apiConfig.mountPoint) return callback(new BoxError(BoxError.FS_ERROR, `${apiConfig.mountPoint} is not mounted`)); } else if (apiConfig.provider === PROVIDER_FILESYSTEM && apiConfig.externalDisk) { if (result.mountpoint === '/') return callback(new BoxError(BoxError.FS_ERROR, `${apiConfig.backupFolder} is not mounted`)); } const needed = 0.6 * used + (1024 * 1024 * 1024); // check if there is atleast 1GB left afterwards. aim for 60% because rsync/tgz won't need full 100% if (result.available <= needed) return callback(new BoxError(BoxError.FS_ERROR, `Not enough disk space for backup. Needed: ${prettyBytes(needed)} Available: ${prettyBytes(result.available)}`)); callback(null); }).catch(function (error) { callback(new BoxError(BoxError.FS_ERROR, error)); }); } function upload(apiConfig, backupFilePath, sourceStream, callback) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof backupFilePath, 'string'); assert.strictEqual(typeof sourceStream, 'object'); assert.strictEqual(typeof callback, 'function'); fs.mkdir(path.dirname(backupFilePath), { recursive: true }, function (error) { if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, error.message)); safe.fs.unlinkSync(backupFilePath); // remove any hardlink var fileStream = fs.createWriteStream(backupFilePath); // this pattern is required to ensure that the file got created before 'finish' fileStream.on('open', function () { sourceStream.pipe(fileStream); }); fileStream.on('error', function (error) { debug('[%s] upload: out stream error.', backupFilePath, error); callback(new BoxError(BoxError.EXTERNAL_ERROR, error.message)); }); fileStream.on('finish', function () { // in test, upload() may or may not be called via sudo script const BACKUP_UID = parseInt(process.env.SUDO_UID, 10) || process.getuid(); // sshfs and cifs handle ownership through the mount args if (apiConfig.provider === PROVIDER_FILESYSTEM || apiConfig.provider === PROVIDER_NFS) { if (!safe.fs.chownSync(backupFilePath, BACKUP_UID, BACKUP_UID)) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Unable to chown:' + safe.error.message)); if (!safe.fs.chownSync(path.dirname(backupFilePath), BACKUP_UID, BACKUP_UID)) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Unable to chown:' + safe.error.message)); } debug('upload %s: done.', backupFilePath); callback(null); }); }); } function download(apiConfig, sourceFilePath, callback) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof sourceFilePath, 'string'); assert.strictEqual(typeof callback, 'function'); debug(`download: ${sourceFilePath}`); if (!safe.fs.existsSync(sourceFilePath)) return callback(new BoxError(BoxError.NOT_FOUND, `File not found: ${sourceFilePath}`)); var fileStream = fs.createReadStream(sourceFilePath); callback(null, fileStream); } function listDir(apiConfig, dir, batchSize, iteratorCallback, callback) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof dir, 'string'); assert.strictEqual(typeof batchSize, 'number'); assert.strictEqual(typeof iteratorCallback, 'function'); assert.strictEqual(typeof callback, 'function'); var entries = []; var entryStream = readdirp(dir, { type: 'files', alwaysStat: true, lstat: true }); entryStream.on('data', function (entryInfo) { if (entryInfo.stats.isSymbolicLink()) return; entries.push({ fullPath: entryInfo.fullPath }); if (entries.length < batchSize) return; entryStream.pause(); iteratorCallback(entries, function (error) { if (error) return callback(error); entries = []; entryStream.resume(); }); }); entryStream.on('warn', function (error) { debug('listDir: warning ', error); }); entryStream.on('end', function () { iteratorCallback(entries, callback); }); } function copy(apiConfig, oldFilePath, newFilePath) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof oldFilePath, 'string'); assert.strictEqual(typeof newFilePath, 'string'); var events = new EventEmitter(); fs.mkdir(path.dirname(newFilePath), { recursive: true }, function (error) { if (error) return events.emit('done', new BoxError(BoxError.EXTERNAL_ERROR, error.message)); events.emit('progress', `Copying ${oldFilePath} to ${newFilePath}`); // sshfs and cifs do not allow preserving attributes var cpOptions = apiConfig.provider === PROVIDER_FILESYSTEM ? '-a' : '-dR'; // this will hardlink backups saving space cpOptions += apiConfig.noHardlinks ? '' : 'l'; shell.spawn('copy', '/bin/cp', [ cpOptions, oldFilePath, newFilePath ], { }, function (error) { if (error) return events.emit('done', new BoxError(BoxError.EXTERNAL_ERROR, error.message)); events.emit('done', null); }); }); return events; } function remove(apiConfig, filename, callback) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof filename, 'string'); assert.strictEqual(typeof callback, 'function'); var stat = safe.fs.statSync(filename); if (!stat) return callback(); if (stat.isFile()) { if (!safe.fs.unlinkSync(filename)) return callback(new BoxError(BoxError.EXTERNAL_ERROR, safe.error.message)); } else if (stat.isDirectory()) { if (!safe.fs.rmdirSync(filename)) return callback(new BoxError(BoxError.EXTERNAL_ERROR, safe.error.message)); } callback(null); } function removeDir(apiConfig, pathPrefix) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof pathPrefix, 'string'); var events = new EventEmitter(); process.nextTick(() => events.emit('progress', `Removing directory ${pathPrefix}`)); shell.spawn('removeDir', '/bin/rm', [ '-rf', pathPrefix ], { }, function (error) { if (error) return events.emit('done', new BoxError(BoxError.EXTERNAL_ERROR, error.message)); events.emit('done', null); }); return events; } function validateBackupTarget(folder) { assert.strictEqual(typeof folder, 'string'); if (path.normalize(folder) !== folder) return new BoxError(BoxError.BAD_FIELD, 'backupFolder must contain a normalized path', { field: 'backupFolder' }); if (!path.isAbsolute(folder)) return new BoxError(BoxError.BAD_FIELD, 'backupFolder must be an absolute path', { field: 'backupFolder' }); if (folder === '/') return new BoxError(BoxError.BAD_FIELD, 'backupFolder cannot be /', { field: 'backupFolder' }); if (!folder.endsWith('/')) folder = folder + '/'; // ensure trailing slash for the prefix matching to work const PROTECTED_PREFIXES = [ '/boot/', '/usr/', '/bin/', '/lib/', '/root/', '/var/lib/', paths.baseDir() ]; if (PROTECTED_PREFIXES.some(p => folder.startsWith(p))) return new BoxError(BoxError.BAD_FIELD, 'backupFolder path is protected', { field: 'backupFolder' }); return null; } function testConfig(apiConfig, callback) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof callback, 'function'); if (apiConfig.provider === PROVIDER_FILESYSTEM) { if (!apiConfig.backupFolder || typeof apiConfig.backupFolder !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'backupFolder must be non-empty string', { field: 'backupFolder' })); let error = validateBackupTarget(apiConfig.backupFolder); if (error) return callback(error); if ('externalDisk' in apiConfig && typeof apiConfig.externalDisk !== 'boolean') return callback(new BoxError(BoxError.BAD_FIELD, 'externalDisk must be boolean', { field: 'externalDisk' })); } if (apiConfig.provider === PROVIDER_SSHFS || apiConfig.provider === PROVIDER_CIFS || apiConfig.provider === PROVIDER_NFS) { if (!apiConfig.mountPoint || typeof apiConfig.mountPoint !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'mountPoint must be non-empty string', { field: 'mountPoint' })); let error = validateBackupTarget(apiConfig.mountPoint); if (error) return callback(error); if (typeof apiConfig.prefix !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'prefix must be a string', { field: 'prefix' })); if (apiConfig.prefix !== '') { if (path.isAbsolute(apiConfig.prefix)) return new BoxError(BoxError.BAD_FIELD, 'prefix must be a relative path', { field: 'backupFolder' }); if (path.normalize(apiConfig.prefix) !== apiConfig.prefix) return callback(new BoxError(BoxError.BAD_FIELD, 'prefix must contain a normalized relative path', { field: 'prefix' })); } const mounts = safe.fs.readFileSync('/proc/mounts', 'utf8'); const mountInfo = mounts.split('\n').filter(function (l) { return l.indexOf(apiConfig.mountPoint) !== -1; })[0]; if (!mountInfo) return callback(new BoxError(BoxError.BAD_FIELD, `${apiConfig.mountPoint} is not mounted`, { field: 'mountPoint' })); if (apiConfig.provider === PROVIDER_SSHFS && !mountInfo.split(' ').find(i => i === 'fuse.sshfs')) return callback(new BoxError(BoxError.BAD_FIELD, 'mountPoint must be a "fuse.sshfs" filesystem', { field: 'mountPoint' })); if (apiConfig.provider === PROVIDER_CIFS && !mountInfo.split(' ').find(i => i === 'cifs')) return callback(new BoxError(BoxError.BAD_FIELD, 'mountPoint must be a "cifs" filesystem', { field: 'mountPoint' })); if (apiConfig.provider === PROVIDER_NFS && !mountInfo.split(' ').find(i => i === 'nfs')) return callback(new BoxError(BoxError.BAD_FIELD, 'mountPoint must be a "nfs" filesystem', { field: 'mountPoint' })); } // common checks const backupPath = getBackupPath(apiConfig); const field = apiConfig.provider === PROVIDER_FILESYSTEM ? 'backupFolder' : 'prefix'; const stat = safe.fs.statSync(backupPath); if (!stat) return callback(new BoxError(BoxError.BAD_FIELD, 'Directory does not exist or cannot be accessed: ' + safe.error.message), { field }); if (!stat.isDirectory()) return callback(new BoxError(BoxError.BAD_FIELD, 'Backup location is not a directory', { field })); if (!safe.fs.mkdirSync(path.join(backupPath, 'snapshot')) && safe.error.code !== 'EEXIST') { if (safe.error && safe.error.code === 'EACCES') return callback(new BoxError(BoxError.BAD_FIELD, `Access denied. Run "chown yellowtent:yellowtent ${backupPath}" on the server`, { field })); return callback(new BoxError(BoxError.BAD_FIELD, safe.error.message, { field })); } if (!safe.fs.writeFileSync(path.join(backupPath, 'cloudron-testfile'), 'testcontent')) { return callback(new BoxError(BoxError.BAD_FIELD, `Unable to create test file as 'yellowtent' user in ${backupPath}: ${safe.error.message}. Check dir/mount permissions`, { field })); } if (!safe.fs.unlinkSync(path.join(backupPath, 'cloudron-testfile'))) { return callback(new BoxError(BoxError.BAD_FIELD, `Unable to remove test file as 'yellowtent' user in ${backupPath}: ${safe.error.message}. Check dir/mount permissions`, { field })); } if ('noHardlinks' in apiConfig && typeof apiConfig.noHardlinks !== 'boolean') return callback(new BoxError(BoxError.BAD_FIELD, 'noHardlinks must be boolean', { field: 'noHardLinks' })); callback(null); } function removePrivateFields(apiConfig) { return apiConfig; } function injectPrivateFields(/* newConfig, currentConfig */) { }