f3d9b81942
autofs mounts are "mounts on demand". this way, instead of mounting lots of things on startup, you can mount it on first access.
326 lines
14 KiB
JavaScript
326 lines
14 KiB
JavaScript
'use strict';
|
|
|
|
exports = module.exports = {
|
|
getBackupPath,
|
|
checkPreconditions,
|
|
|
|
upload,
|
|
download,
|
|
|
|
copy,
|
|
|
|
exists,
|
|
listDir,
|
|
|
|
remove,
|
|
removeDir,
|
|
|
|
testConfig,
|
|
removePrivateFields,
|
|
injectPrivateFields
|
|
};
|
|
|
|
const PROVIDER_FILESYSTEM = 'filesystem';
|
|
const PROVIDER_SSHFS = 'sshfs';
|
|
const PROVIDER_CIFS = 'cifs';
|
|
const PROVIDER_NFS = 'nfs';
|
|
|
|
var assert = require('assert'),
|
|
BoxError = require('../boxerror.js'),
|
|
DataLayout = require('../datalayout.js'),
|
|
debug = require('debug')('box:storage/filesystem'),
|
|
df = require('@sindresorhus/df'),
|
|
EventEmitter = require('events'),
|
|
fs = require('fs'),
|
|
path = require('path'),
|
|
paths = require('../paths.js'),
|
|
prettyBytes = require('pretty-bytes'),
|
|
readdirp = require('readdirp'),
|
|
safe = require('safetydance'),
|
|
shell = require('../shell.js');
|
|
|
|
// storage api
|
|
function getBackupPath(apiConfig) {
|
|
assert.strictEqual(typeof apiConfig, 'object');
|
|
|
|
if (apiConfig.provider === PROVIDER_SSHFS) return path.join(apiConfig.mountPoint, apiConfig.prefix);
|
|
if (apiConfig.provider === PROVIDER_CIFS) return path.join(apiConfig.mountPoint, apiConfig.prefix);
|
|
if (apiConfig.provider === PROVIDER_NFS) return path.join(apiConfig.mountPoint, apiConfig.prefix);
|
|
|
|
return apiConfig.backupFolder;
|
|
}
|
|
|
|
// the du call in the function below requires root
|
|
function checkPreconditions(apiConfig, dataLayout, callback) {
|
|
assert.strictEqual(typeof apiConfig, 'object');
|
|
assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
|
|
assert.strictEqual(typeof callback, 'function');
|
|
|
|
let used = 0;
|
|
for (let localPath of dataLayout.localPaths()) {
|
|
debug(`checkPreconditions: getting disk usage of ${localPath}`);
|
|
let result = safe.child_process.execSync(`du -Dsb ${localPath}`, { encoding: 'utf8' });
|
|
if (!result) return callback(new BoxError(BoxError.FS_ERROR, safe.error));
|
|
used += parseInt(result, 10);
|
|
}
|
|
|
|
debug(`checkPreconditions: ${used} bytes`);
|
|
|
|
df.file(getBackupPath(apiConfig)).then(function (result) {
|
|
|
|
// Check filesystem is mounted so we don't write into the actual folder on disk
|
|
if (apiConfig.provider === PROVIDER_SSHFS || apiConfig.provider === PROVIDER_CIFS || apiConfig.provider === PROVIDER_NFS) {
|
|
if (result.mountpoint !== apiConfig.mountPoint) return callback(new BoxError(BoxError.FS_ERROR, `${apiConfig.mountPoint} is not mounted`));
|
|
} else if (apiConfig.provider === PROVIDER_FILESYSTEM && apiConfig.externalDisk) {
|
|
if (result.mountpoint === '/') return callback(new BoxError(BoxError.FS_ERROR, `${apiConfig.backupFolder} is not mounted`));
|
|
}
|
|
|
|
const needed = 0.6 * used + (1024 * 1024 * 1024); // check if there is atleast 1GB left afterwards. aim for 60% because rsync/tgz won't need full 100%
|
|
if (result.available <= needed) return callback(new BoxError(BoxError.FS_ERROR, `Not enough disk space for backup. Needed: ${prettyBytes(needed)} Available: ${prettyBytes(result.available)}`));
|
|
|
|
callback(null);
|
|
}).catch(function (error) {
|
|
callback(new BoxError(BoxError.FS_ERROR, error));
|
|
});
|
|
}
|
|
|
|
function upload(apiConfig, backupFilePath, sourceStream, callback) {
|
|
assert.strictEqual(typeof apiConfig, 'object');
|
|
assert.strictEqual(typeof backupFilePath, 'string');
|
|
assert.strictEqual(typeof sourceStream, 'object');
|
|
assert.strictEqual(typeof callback, 'function');
|
|
|
|
fs.mkdir(path.dirname(backupFilePath), { recursive: true }, function (error) {
|
|
if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, error.message));
|
|
|
|
safe.fs.unlinkSync(backupFilePath); // remove any hardlink
|
|
|
|
var fileStream = fs.createWriteStream(backupFilePath);
|
|
|
|
// this pattern is required to ensure that the file got created before 'finish'
|
|
fileStream.on('open', function () {
|
|
sourceStream.pipe(fileStream);
|
|
});
|
|
|
|
fileStream.on('error', function (error) {
|
|
debug('[%s] upload: out stream error.', backupFilePath, error);
|
|
callback(new BoxError(BoxError.EXTERNAL_ERROR, error.message));
|
|
});
|
|
|
|
fileStream.on('finish', function () {
|
|
// in test, upload() may or may not be called via sudo script
|
|
const BACKUP_UID = parseInt(process.env.SUDO_UID, 10) || process.getuid();
|
|
|
|
// sshfs and cifs handle ownership through the mount args
|
|
if (apiConfig.provider === PROVIDER_FILESYSTEM || apiConfig.provider === PROVIDER_NFS) {
|
|
if (!safe.fs.chownSync(backupFilePath, BACKUP_UID, BACKUP_UID)) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Unable to chown:' + safe.error.message));
|
|
if (!safe.fs.chownSync(path.dirname(backupFilePath), BACKUP_UID, BACKUP_UID)) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Unable to chown:' + safe.error.message));
|
|
}
|
|
|
|
debug('upload %s: done.', backupFilePath);
|
|
|
|
callback(null);
|
|
});
|
|
});
|
|
}
|
|
|
|
function download(apiConfig, sourceFilePath, callback) {
|
|
assert.strictEqual(typeof apiConfig, 'object');
|
|
assert.strictEqual(typeof sourceFilePath, 'string');
|
|
assert.strictEqual(typeof callback, 'function');
|
|
|
|
debug(`download: ${sourceFilePath}`);
|
|
|
|
if (!safe.fs.existsSync(sourceFilePath)) return callback(new BoxError(BoxError.NOT_FOUND, `File not found: ${sourceFilePath}`));
|
|
|
|
var fileStream = fs.createReadStream(sourceFilePath);
|
|
callback(null, fileStream);
|
|
}
|
|
|
|
function exists(apiConfig, sourceFilePath, callback) {
|
|
assert.strictEqual(typeof apiConfig, 'object');
|
|
assert.strictEqual(typeof sourceFilePath, 'string');
|
|
assert.strictEqual(typeof callback, 'function');
|
|
|
|
// do not use existsSync because it does not return EPERM etc
|
|
if (!safe.fs.statSync(sourceFilePath)) {
|
|
if (safe.error && safe.error.code === 'ENOENT') return callback(null, false);
|
|
if (safe.error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Exists ${sourceFilePath}: ${safe.error.message}`));
|
|
}
|
|
|
|
callback(null, true);
|
|
}
|
|
|
|
function listDir(apiConfig, dir, batchSize, iteratorCallback, callback) {
|
|
assert.strictEqual(typeof apiConfig, 'object');
|
|
assert.strictEqual(typeof dir, 'string');
|
|
assert.strictEqual(typeof batchSize, 'number');
|
|
assert.strictEqual(typeof iteratorCallback, 'function');
|
|
assert.strictEqual(typeof callback, 'function');
|
|
|
|
var entries = [];
|
|
var entryStream = readdirp(dir, { type: 'files', alwaysStat: true, lstat: true });
|
|
entryStream.on('data', function (entryInfo) {
|
|
if (entryInfo.stats.isSymbolicLink()) return;
|
|
|
|
entries.push({ fullPath: entryInfo.fullPath });
|
|
if (entries.length < batchSize) return;
|
|
entryStream.pause();
|
|
iteratorCallback(entries, function (error) {
|
|
if (error) return callback(error);
|
|
|
|
entries = [];
|
|
entryStream.resume();
|
|
});
|
|
});
|
|
entryStream.on('warn', function (error) {
|
|
debug('listDir: warning ', error);
|
|
});
|
|
entryStream.on('end', function () {
|
|
iteratorCallback(entries, callback);
|
|
});
|
|
}
|
|
|
|
function copy(apiConfig, oldFilePath, newFilePath) {
|
|
assert.strictEqual(typeof apiConfig, 'object');
|
|
assert.strictEqual(typeof oldFilePath, 'string');
|
|
assert.strictEqual(typeof newFilePath, 'string');
|
|
|
|
var events = new EventEmitter();
|
|
|
|
fs.mkdir(path.dirname(newFilePath), { recursive: true }, function (error) {
|
|
if (error) return events.emit('done', new BoxError(BoxError.EXTERNAL_ERROR, error.message));
|
|
|
|
events.emit('progress', `Copying ${oldFilePath} to ${newFilePath}`);
|
|
|
|
// sshfs and cifs do not allow preserving attributes
|
|
var cpOptions = apiConfig.provider === PROVIDER_FILESYSTEM ? '-a' : '-dR';
|
|
|
|
// this will hardlink backups saving space
|
|
cpOptions += apiConfig.noHardlinks ? '' : 'l';
|
|
|
|
shell.spawn('copy', '/bin/cp', [ cpOptions, oldFilePath, newFilePath ], { }, function (error) {
|
|
if (error) return events.emit('done', new BoxError(BoxError.EXTERNAL_ERROR, error.message));
|
|
|
|
events.emit('done', null);
|
|
});
|
|
});
|
|
|
|
return events;
|
|
}
|
|
|
|
function remove(apiConfig, filename, callback) {
|
|
assert.strictEqual(typeof apiConfig, 'object');
|
|
assert.strictEqual(typeof filename, 'string');
|
|
assert.strictEqual(typeof callback, 'function');
|
|
|
|
var stat = safe.fs.statSync(filename);
|
|
if (!stat) return callback();
|
|
|
|
if (stat.isFile()) {
|
|
if (!safe.fs.unlinkSync(filename)) return callback(new BoxError(BoxError.EXTERNAL_ERROR, safe.error.message));
|
|
} else if (stat.isDirectory()) {
|
|
if (!safe.fs.rmdirSync(filename)) return callback(new BoxError(BoxError.EXTERNAL_ERROR, safe.error.message));
|
|
}
|
|
|
|
callback(null);
|
|
}
|
|
|
|
function removeDir(apiConfig, pathPrefix) {
|
|
assert.strictEqual(typeof apiConfig, 'object');
|
|
assert.strictEqual(typeof pathPrefix, 'string');
|
|
|
|
var events = new EventEmitter();
|
|
|
|
process.nextTick(() => events.emit('progress', `Removing directory ${pathPrefix}`));
|
|
|
|
shell.spawn('removeDir', '/bin/rm', [ '-rf', pathPrefix ], { }, function (error) {
|
|
if (error) return events.emit('done', new BoxError(BoxError.EXTERNAL_ERROR, error.message));
|
|
|
|
events.emit('done', null);
|
|
});
|
|
|
|
return events;
|
|
}
|
|
|
|
function validateBackupTarget(folder) {
|
|
assert.strictEqual(typeof folder, 'string');
|
|
|
|
if (path.normalize(folder) !== folder) return new BoxError(BoxError.BAD_FIELD, 'backupFolder must contain a normalized path', { field: 'backupFolder' });
|
|
if (!path.isAbsolute(folder)) return new BoxError(BoxError.BAD_FIELD, 'backupFolder must be an absolute path', { field: 'backupFolder' });
|
|
|
|
if (folder === '/') return new BoxError(BoxError.BAD_FIELD, 'backupFolder cannot be /', { field: 'backupFolder' });
|
|
|
|
if (!folder.endsWith('/')) folder = folder + '/'; // ensure trailing slash for the prefix matching to work
|
|
const PROTECTED_PREFIXES = [ '/boot/', '/usr/', '/bin/', '/lib/', '/root/', '/var/lib/', paths.baseDir() ];
|
|
|
|
if (PROTECTED_PREFIXES.some(p => folder.startsWith(p))) return new BoxError(BoxError.BAD_FIELD, 'backupFolder path is protected', { field: 'backupFolder' });
|
|
|
|
return null;
|
|
}
|
|
|
|
function testConfig(apiConfig, callback) {
|
|
assert.strictEqual(typeof apiConfig, 'object');
|
|
assert.strictEqual(typeof callback, 'function');
|
|
|
|
if (apiConfig.provider === PROVIDER_FILESYSTEM) {
|
|
if (!apiConfig.backupFolder || typeof apiConfig.backupFolder !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'backupFolder must be non-empty string', { field: 'backupFolder' }));
|
|
let error = validateBackupTarget(apiConfig.backupFolder);
|
|
if (error) return callback(error);
|
|
|
|
if ('externalDisk' in apiConfig && typeof apiConfig.externalDisk !== 'boolean') return callback(new BoxError(BoxError.BAD_FIELD, 'externalDisk must be boolean', { field: 'externalDisk' }));
|
|
}
|
|
|
|
if (apiConfig.provider === PROVIDER_SSHFS || apiConfig.provider === PROVIDER_CIFS || apiConfig.provider === PROVIDER_NFS) {
|
|
if (!apiConfig.mountPoint || typeof apiConfig.mountPoint !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'mountPoint must be non-empty string', { field: 'mountPoint' }));
|
|
let error = validateBackupTarget(apiConfig.mountPoint);
|
|
if (error) return callback(error);
|
|
|
|
if (typeof apiConfig.prefix !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'prefix must be a string', { field: 'prefix' }));
|
|
if (apiConfig.prefix !== '') {
|
|
if (path.isAbsolute(apiConfig.prefix)) return new BoxError(BoxError.BAD_FIELD, 'prefix must be a relative path', { field: 'backupFolder' });
|
|
if (path.normalize(apiConfig.prefix) !== apiConfig.prefix) return callback(new BoxError(BoxError.BAD_FIELD, 'prefix must contain a normalized relative path', { field: 'prefix' }));
|
|
}
|
|
|
|
const mounts = safe.fs.readFileSync('/proc/mounts', 'utf8');
|
|
const mountInfo = mounts.split('\n').filter(function (l) { return l.indexOf(apiConfig.mountPoint) !== -1; })[0];
|
|
if (!mountInfo) return callback(new BoxError(BoxError.BAD_FIELD, `${apiConfig.mountPoint} is not mounted`, { field: 'mountPoint' }));
|
|
|
|
if (apiConfig.provider === PROVIDER_SSHFS && !mountInfo.split(' ').find(i => i === 'fuse.sshfs' || i === 'autofs')) return callback(new BoxError(BoxError.BAD_FIELD, 'mountPoint must be a "fuse.sshfs" filesystem', { field: 'mountPoint' }));
|
|
if (apiConfig.provider === PROVIDER_CIFS && !mountInfo.split(' ').find(i => i === 'cifs' || i === 'autofs')) return callback(new BoxError(BoxError.BAD_FIELD, 'mountPoint must be a "cifs" filesystem', { field: 'mountPoint' }));
|
|
if (apiConfig.provider === PROVIDER_NFS && !mountInfo.split(' ').find(i => i === 'nfs' || i === 'nfs4' || i === 'autofs')) return callback(new BoxError(BoxError.BAD_FIELD, 'mountPoint must be a "nfs" filesystem', { field: 'mountPoint' }));
|
|
}
|
|
|
|
// common checks
|
|
const backupPath = getBackupPath(apiConfig);
|
|
const field = apiConfig.provider === PROVIDER_FILESYSTEM ? 'backupFolder' : 'prefix';
|
|
|
|
const stat = safe.fs.statSync(backupPath);
|
|
if (!stat) return callback(new BoxError(BoxError.BAD_FIELD, 'Directory does not exist or cannot be accessed: ' + safe.error.message), { field });
|
|
if (!stat.isDirectory()) return callback(new BoxError(BoxError.BAD_FIELD, 'Backup location is not a directory', { field }));
|
|
|
|
if (!safe.fs.mkdirSync(path.join(backupPath, 'snapshot')) && safe.error.code !== 'EEXIST') {
|
|
if (safe.error && safe.error.code === 'EACCES') return callback(new BoxError(BoxError.BAD_FIELD, `Access denied. Run "chown yellowtent:yellowtent ${backupPath}" on the server`, { field }));
|
|
return callback(new BoxError(BoxError.BAD_FIELD, safe.error.message, { field }));
|
|
}
|
|
|
|
if (!safe.fs.writeFileSync(path.join(backupPath, 'cloudron-testfile'), 'testcontent')) {
|
|
return callback(new BoxError(BoxError.BAD_FIELD, `Unable to create test file as 'yellowtent' user in ${backupPath}: ${safe.error.message}. Check dir/mount permissions`, { field }));
|
|
}
|
|
|
|
if (!safe.fs.unlinkSync(path.join(backupPath, 'cloudron-testfile'))) {
|
|
return callback(new BoxError(BoxError.BAD_FIELD, `Unable to remove test file as 'yellowtent' user in ${backupPath}: ${safe.error.message}. Check dir/mount permissions`, { field }));
|
|
}
|
|
|
|
if ('noHardlinks' in apiConfig && typeof apiConfig.noHardlinks !== 'boolean') return callback(new BoxError(BoxError.BAD_FIELD, 'noHardlinks must be boolean', { field: 'noHardLinks' }));
|
|
|
|
callback(null);
|
|
}
|
|
|
|
function removePrivateFields(apiConfig) {
|
|
return apiConfig;
|
|
}
|
|
|
|
function injectPrivateFields(/* newConfig, currentConfig */) {
|
|
}
|