282 lines
12 KiB
JavaScript
282 lines
12 KiB
JavaScript
'use strict';
|
|
|
|
exports = module.exports = {
|
|
getBackupPath: getBackupPath,
|
|
checkPreconditions: checkPreconditions,
|
|
|
|
upload: upload,
|
|
download: download,
|
|
|
|
copy: copy,
|
|
|
|
listDir: listDir,
|
|
|
|
remove: remove,
|
|
removeDir: removeDir,
|
|
|
|
testConfig: testConfig,
|
|
removePrivateFields: removePrivateFields,
|
|
injectPrivateFields: injectPrivateFields
|
|
};
|
|
|
|
const PROVIDER_FILESYSTEM = 'filesystem';
|
|
const PROVIDER_SSHFS = 'sshfs';
|
|
const PROVIDER_CIFS = 'cifs';
|
|
const PROVIDER_NFS = 'nfs';
|
|
|
|
var assert = require('assert'),
|
|
BoxError = require('../boxerror.js'),
|
|
DataLayout = require('../datalayout.js'),
|
|
debug = require('debug')('box:storage/filesystem'),
|
|
df = require('@sindresorhus/df'),
|
|
EventEmitter = require('events'),
|
|
fs = require('fs'),
|
|
path = require('path'),
|
|
prettyBytes = require('pretty-bytes'),
|
|
readdirp = require('readdirp'),
|
|
safe = require('safetydance'),
|
|
shell = require('../shell.js');
|
|
|
|
// storage api
|
|
function getBackupPath(apiConfig) {
|
|
assert.strictEqual(typeof apiConfig, 'object');
|
|
|
|
if (apiConfig.provider === PROVIDER_SSHFS) return path.join(apiConfig.mountPoint, apiConfig.prefix);
|
|
if (apiConfig.provider === PROVIDER_CIFS) return path.join(apiConfig.mountPoint, apiConfig.prefix);
|
|
if (apiConfig.provider === PROVIDER_NFS) return path.join(apiConfig.mountPoint, apiConfig.prefix);
|
|
|
|
return apiConfig.backupFolder;
|
|
}
|
|
|
|
// the du call in the function below requires root
|
|
function checkPreconditions(apiConfig, dataLayout, callback) {
|
|
assert.strictEqual(typeof apiConfig, 'object');
|
|
assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
|
|
assert.strictEqual(typeof callback, 'function');
|
|
|
|
let used = 0;
|
|
for (let localPath of dataLayout.localPaths()) {
|
|
debug(`checkPreconditions: getting disk usage of ${localPath}`);
|
|
let result = safe.child_process.execSync(`du -Dsb ${localPath}`, { encoding: 'utf8' });
|
|
if (!result) return callback(new BoxError(BoxError.FS_ERROR, safe.error));
|
|
used += parseInt(result, 10);
|
|
}
|
|
|
|
debug(`checkPreconditions: ${used} bytes`);
|
|
|
|
df.file(getBackupPath(apiConfig)).then(function (result) {
|
|
|
|
// Check filesystem is mounted so we don't write into the actual folder on disk
|
|
if (apiConfig.provider === PROVIDER_SSHFS || apiConfig.provider === PROVIDER_CIFS || apiConfig.provider === PROVIDER_NFS) {
|
|
if (result.mountpoint !== apiConfig.mountPoint) return callback(new BoxError(BoxError.FS_ERROR, `${apiConfig.mountPoint} is not mounted`));
|
|
} else if (apiConfig.provider === PROVIDER_FILESYSTEM && apiConfig.externalDisk) {
|
|
if (result.mountpoint === '/') return callback(new BoxError(BoxError.FS_ERROR, `${apiConfig.backupFolder} is not mounted`));
|
|
}
|
|
|
|
const needed = used + (1024 * 1024 * 1024); // check if there is atleast 1GB left afterwards
|
|
if (result.available <= needed) return callback(new BoxError(BoxError.FS_ERROR, `Not enough disk space for backup. Needed: ${prettyBytes(needed)} Available: ${prettyBytes(result.available)}`));
|
|
|
|
callback(null);
|
|
}).catch(function (error) {
|
|
callback(new BoxError(BoxError.FS_ERROR, error));
|
|
});
|
|
}
|
|
|
|
function upload(apiConfig, backupFilePath, sourceStream, callback) {
|
|
assert.strictEqual(typeof apiConfig, 'object');
|
|
assert.strictEqual(typeof backupFilePath, 'string');
|
|
assert.strictEqual(typeof sourceStream, 'object');
|
|
assert.strictEqual(typeof callback, 'function');
|
|
|
|
fs.mkdir(path.dirname(backupFilePath), { recursive: true }, function (error) {
|
|
if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, error.message));
|
|
|
|
safe.fs.unlinkSync(backupFilePath); // remove any hardlink
|
|
|
|
var fileStream = fs.createWriteStream(backupFilePath);
|
|
|
|
// this pattern is required to ensure that the file got created before 'finish'
|
|
fileStream.on('open', function () {
|
|
sourceStream.pipe(fileStream);
|
|
});
|
|
|
|
fileStream.on('error', function (error) {
|
|
debug('[%s] upload: out stream error.', backupFilePath, error);
|
|
callback(new BoxError(BoxError.EXTERNAL_ERROR, error.message));
|
|
});
|
|
|
|
fileStream.on('finish', function () {
|
|
// in test, upload() may or may not be called via sudo script
|
|
const BACKUP_UID = parseInt(process.env.SUDO_UID, 10) || process.getuid();
|
|
|
|
// sshfs and cifs handle ownership through the mount args
|
|
if (apiConfig.provider === PROVIDER_FILESYSTEM && !safe.fs.chownSync(backupFilePath, BACKUP_UID, BACKUP_UID)) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Unable to chown:' + safe.error.message));
|
|
if (apiConfig.provider === PROVIDER_FILESYSTEM && !safe.fs.chownSync(path.dirname(backupFilePath), BACKUP_UID, BACKUP_UID)) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Unable to chown:' + safe.error.message));
|
|
|
|
debug('upload %s: done.', backupFilePath);
|
|
|
|
callback(null);
|
|
});
|
|
});
|
|
}
|
|
|
|
function download(apiConfig, sourceFilePath, callback) {
|
|
assert.strictEqual(typeof apiConfig, 'object');
|
|
assert.strictEqual(typeof sourceFilePath, 'string');
|
|
assert.strictEqual(typeof callback, 'function');
|
|
|
|
debug(`download: ${sourceFilePath}`);
|
|
|
|
if (!safe.fs.existsSync(sourceFilePath)) return callback(new BoxError(BoxError.NOT_FOUND, `File not found: ${sourceFilePath}`));
|
|
|
|
var fileStream = fs.createReadStream(sourceFilePath);
|
|
callback(null, fileStream);
|
|
}
|
|
|
|
function listDir(apiConfig, dir, batchSize, iteratorCallback, callback) {
|
|
assert.strictEqual(typeof apiConfig, 'object');
|
|
assert.strictEqual(typeof dir, 'string');
|
|
assert.strictEqual(typeof batchSize, 'number');
|
|
assert.strictEqual(typeof iteratorCallback, 'function');
|
|
assert.strictEqual(typeof callback, 'function');
|
|
|
|
var entries = [];
|
|
var entryStream = readdirp(dir, { type: 'files', alwaysStat: true, lstat: true });
|
|
entryStream.on('data', function (entryInfo) {
|
|
if (entryInfo.stats.isSymbolicLink()) return;
|
|
|
|
entries.push({ fullPath: entryInfo.fullPath });
|
|
if (entries.length < batchSize) return;
|
|
entryStream.pause();
|
|
iteratorCallback(entries, function (error) {
|
|
if (error) return callback(error);
|
|
|
|
entries = [];
|
|
entryStream.resume();
|
|
});
|
|
});
|
|
entryStream.on('warn', function (error) {
|
|
debug('listDir: warning ', error);
|
|
});
|
|
entryStream.on('end', function () {
|
|
iteratorCallback(entries, callback);
|
|
});
|
|
}
|
|
|
|
function copy(apiConfig, oldFilePath, newFilePath) {
|
|
assert.strictEqual(typeof apiConfig, 'object');
|
|
assert.strictEqual(typeof oldFilePath, 'string');
|
|
assert.strictEqual(typeof newFilePath, 'string');
|
|
|
|
var events = new EventEmitter();
|
|
|
|
fs.mkdir(path.dirname(newFilePath), { recursive: true }, function (error) {
|
|
if (error) return events.emit('done', new BoxError(BoxError.EXTERNAL_ERROR, error.message));
|
|
|
|
events.emit('progress', `Copying ${oldFilePath} to ${newFilePath}`);
|
|
|
|
// sshfs and cifs do not allow preserving attributes
|
|
var cpOptions = apiConfig.provider === PROVIDER_FILESYSTEM ? '-a' : '-dR';
|
|
|
|
// this will hardlink backups saving space
|
|
cpOptions += apiConfig.noHardlinks ? '' : 'l';
|
|
|
|
shell.spawn('copy', '/bin/cp', [ cpOptions, oldFilePath, newFilePath ], { }, function (error) {
|
|
if (error) return events.emit('done', new BoxError(BoxError.EXTERNAL_ERROR, error.message));
|
|
|
|
events.emit('done', null);
|
|
});
|
|
});
|
|
|
|
return events;
|
|
}
|
|
|
|
function remove(apiConfig, filename, callback) {
|
|
assert.strictEqual(typeof apiConfig, 'object');
|
|
assert.strictEqual(typeof filename, 'string');
|
|
assert.strictEqual(typeof callback, 'function');
|
|
|
|
var stat = safe.fs.statSync(filename);
|
|
if (!stat) return callback();
|
|
|
|
if (stat.isFile()) {
|
|
if (!safe.fs.unlinkSync(filename)) return callback(new BoxError(BoxError.EXTERNAL_ERROR, safe.error.message));
|
|
} else if (stat.isDirectory()) {
|
|
if (!safe.fs.rmdirSync(filename)) return callback(new BoxError(BoxError.EXTERNAL_ERROR, safe.error.message));
|
|
}
|
|
|
|
callback(null);
|
|
}
|
|
|
|
function removeDir(apiConfig, pathPrefix) {
|
|
assert.strictEqual(typeof apiConfig, 'object');
|
|
assert.strictEqual(typeof pathPrefix, 'string');
|
|
|
|
var events = new EventEmitter();
|
|
|
|
process.nextTick(() => events.emit('progress', `Removing directory ${pathPrefix}`));
|
|
|
|
shell.spawn('removeDir', '/bin/rm', [ '-rf', pathPrefix ], { }, function (error) {
|
|
if (error) return events.emit('done', new BoxError(BoxError.EXTERNAL_ERROR, error.message));
|
|
|
|
events.emit('done', null);
|
|
});
|
|
|
|
return events;
|
|
}
|
|
|
|
function testConfig(apiConfig, callback) {
|
|
assert.strictEqual(typeof apiConfig, 'object');
|
|
assert.strictEqual(typeof callback, 'function');
|
|
|
|
if (apiConfig.provider === PROVIDER_FILESYSTEM) {
|
|
if (!apiConfig.backupFolder || typeof apiConfig.backupFolder !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'backupFolder must be non-empty string', { field: 'backupFolder' }));
|
|
if ('externalDisk' in apiConfig && typeof apiConfig.externalDisk !== 'boolean') return callback(new BoxError(BoxError.BAD_FIELD, 'externalDisk must be boolean', { field: 'externalDisk' }));
|
|
}
|
|
|
|
if (apiConfig.provider === PROVIDER_SSHFS || apiConfig.provider === PROVIDER_CIFS || apiConfig.provider === PROVIDER_NFS) {
|
|
if (!apiConfig.mountPoint || typeof apiConfig.mountPoint !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'mountPoint must be non-empty string', { field: 'mountPoint' }));
|
|
if (typeof apiConfig.prefix !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'prefix must be a string', { field: 'prefix' }));
|
|
|
|
const mounts = safe.fs.readFileSync('/proc/mounts', 'utf8');
|
|
const mountInfo = mounts.split('\n').filter(function (l) { return l.indexOf(apiConfig.mountPoint) !== -1; })[0];
|
|
if (!mountInfo) return callback(new BoxError(BoxError.BAD_FIELD, `${apiConfig.mountPoint} is not mounted`, { field: 'mountPoint' }));
|
|
|
|
if (apiConfig.provider === PROVIDER_SSHFS && !mountInfo.split(' ').find(i => i === 'fuse.sshfs')) return callback(new BoxError(BoxError.BAD_FIELD, 'mountPoint must be a "fuse.sshfs" filesystem', { field: 'mountPoint' }));
|
|
if (apiConfig.provider === PROVIDER_CIFS && !mountInfo.split(' ').find(i => i === 'cifs')) return callback(new BoxError(BoxError.BAD_FIELD, 'mountPoint must be a "cifs" filesystem', { field: 'mountPoint' }));
|
|
if (apiConfig.provider === PROVIDER_NFS && !mountInfo.split(' ').find(i => i === 'nfs')) return callback(new BoxError(BoxError.BAD_FIELD, 'mountPoint must be a "nfs" filesystem', { field: 'mountPoint' }));
|
|
}
|
|
|
|
// common checks
|
|
const backupPath = getBackupPath(apiConfig);
|
|
const field = apiConfig.provider === PROVIDER_FILESYSTEM ? 'backupFolder' : 'prefix';
|
|
|
|
const stat = safe.fs.statSync(backupPath);
|
|
if (!stat) return callback(new BoxError(BoxError.BAD_FIELD, 'Directory does not exist or cannot be accessed: ' + safe.error.message), { field });
|
|
if (!stat.isDirectory()) return callback(new BoxError(BoxError.BAD_FIELD, 'Backup location is not a directory', { field }));
|
|
|
|
if (!safe.fs.mkdirSync(path.join(backupPath, 'snapshot')) && safe.error.code !== 'EEXIST') {
|
|
if (safe.error && safe.error.code === 'EACCES') return callback(new BoxError(BoxError.BAD_FIELD, `Access denied. Run "chown yellowtent:yellowtent ${backupPath}" on the server`, { field }));
|
|
return callback(new BoxError(BoxError.BAD_FIELD, safe.error.message, { field }));
|
|
}
|
|
|
|
if (!safe.fs.writeFileSync(path.join(backupPath, 'cloudron-testfile'), 'testcontent')) {
|
|
return callback(new BoxError(BoxError.BAD_FIELD, `Unable to create test file as 'yellowtent' user in ${backupPath}: ${safe.error.message}. Check dir/mount permissions`, { field }));
|
|
}
|
|
|
|
if (!safe.fs.unlinkSync(path.join(backupPath, 'cloudron-testfile'))) {
|
|
return callback(new BoxError(BoxError.BAD_FIELD, `Unable to remove test file as 'yellowtent' user in ${backupPath}: ${safe.error.message}. Check dir/mount permissions`, { field }));
|
|
}
|
|
|
|
if ('noHardlinks' in apiConfig && typeof apiConfig.noHardlinks !== 'boolean') return callback(new BoxError(BoxError.BAD_FIELD, 'noHardlinks must be boolean', { field: 'noHardLinks' }));
|
|
|
|
callback(null);
|
|
}
|
|
|
|
function removePrivateFields(apiConfig) {
|
|
return apiConfig;
|
|
}
|
|
|
|
function injectPrivateFields(/* newConfig, currentConfig */) {
|
|
}
|