cloudron-box/src/storage/filesystem.js

'use strict';

exports = module.exports = {
    getBackupRootPath,
    checkBackupPreconditions,

    upload,
    download,

    copy,

    exists,
    listDir,

    remove,
    removeDir,

    remount,

    testConfig,
    removePrivateFields,
    injectPrivateFields
};

const PROVIDER_FILESYSTEM = 'filesystem';
const PROVIDER_MOUNTPOINT = 'mountpoint';
const PROVIDER_SSHFS = 'sshfs';
const PROVIDER_CIFS = 'cifs';
const PROVIDER_XFS = 'xfs';
const PROVIDER_NFS = 'nfs';
const PROVIDER_EXT4 = 'ext4';

const assert = require('assert'),
    BoxError = require('../boxerror.js'),
    constants = require('../constants.js'),
    DataLayout = require('../datalayout.js'),
    debug = require('debug')('box:storage/filesystem'),
    df = require('@sindresorhus/df'),
    fs = require('fs'),
    mounts = require('../mounts.js'),
    path = require('path'),
    paths = require('../paths.js'),
    readdirp = require('readdirp'),
    safe = require('safetydance'),
    shell = require('../shell.js');

// storage api
function getBackupRootPath(apiConfig) {
    assert.strictEqual(typeof apiConfig, 'object');

    switch (apiConfig.provider) {
    case PROVIDER_SSHFS:
    case PROVIDER_NFS:
    case PROVIDER_CIFS:
    case PROVIDER_EXT4:
    case PROVIDER_XFS:
        return path.join(paths.MANAGED_BACKUP_MOUNT_DIR, apiConfig.prefix);
    case PROVIDER_MOUNTPOINT:
        return path.join(apiConfig.mountPoint, apiConfig.prefix);
    case PROVIDER_FILESYSTEM:
        return apiConfig.backupFolder;
    }
}

// binary units (non SI) 1024 based
function prettyBytes(bytes) {
    assert.strictEqual(typeof bytes, 'number');

    const i = Math.floor(Math.log(bytes) / Math.log(1024)),
        sizes = ['B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'];

    return (bytes / Math.pow(1024, i)).toFixed(2) * 1 + '' + sizes[i];
}

// the du call in the function below requires root
async function checkBackupPreconditions(apiConfig, dataLayout) {
    assert.strictEqual(typeof apiConfig, 'object');
    assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');

    // Check filesystem is mounted so we don't write into the actual folder on disk
    if (mounts.isManagedProvider(apiConfig.provider) || apiConfig.provider === 'mountpoint') {
        const hostPath = mounts.isManagedProvider(apiConfig.provider) ? paths.MANAGED_BACKUP_MOUNT_DIR : apiConfig.mountPoint;
        const status = await mounts.getStatus(apiConfig.provider, hostPath); // { state, message }
        debug(`clean: mount point status is ${JSON.stringify(status)}`);
        if (status.state !== 'active') throw new BoxError(BoxError.MOUNT_ERROR, `Backup endpoint is not mounted: ${status.message}`);
    }

    const [error, dfResult] = await safe(df.file(getBackupRootPath(apiConfig)));
    if (error) throw new BoxError(BoxError.FS_ERROR, `Error when checking for disk space: ${error.message}`);

    let used = 0;
    for (const localPath of dataLayout.localPaths()) {
        debug(`checkBackupPreconditions: getting disk usage of ${localPath}`);
        const result = safe.child_process.execSync(`du -Dsb ${localPath}`, { encoding: 'utf8' });
        if (!result) throw new BoxError(BoxError.FS_ERROR, `du error: ${safe.error.message}`);
        used += parseInt(result, 10);
    }

    debug(`checkBackupPreconditions: ${used} bytes`);

    const needed = 0.6 * used + (1024 * 1024 * 1024); // check if there is atleast 1GB left afterwards. aim for 60% because rsync/tgz won't need full 100%
    if (dfResult.available <= needed) throw new BoxError(BoxError.FS_ERROR, `Not enough disk space for backup. Needed: ${prettyBytes(needed)} Available: ${prettyBytes(dfResult.available)}`);
}

function hasChownSupportSync(apiConfig) {
    switch (apiConfig.provider) {
    case PROVIDER_NFS:
    case PROVIDER_EXT4:
    case PROVIDER_XFS:
    case PROVIDER_FILESYSTEM:
        return true;
    case PROVIDER_SSHFS:
        // sshfs can be mounted as root or normal user. when mounted as root, we have to chown since we remove backups as the yellowtent user
        // when mounted as non-root user, files are created as yellowtent user but they are still owned by the non-root user (thus del also works)
        return apiConfig.mountOptions.user === 'root';
    case PROVIDER_CIFS:
        return true;
    case PROVIDER_MOUNTPOINT:
        return apiConfig.chown;
    }
}

function upload(apiConfig, backupFilePath, sourceStream, callback) {
    assert.strictEqual(typeof apiConfig, 'object');
    assert.strictEqual(typeof backupFilePath, 'string');
    assert.strictEqual(typeof sourceStream, 'object');
    assert.strictEqual(typeof callback, 'function');

    fs.mkdir(path.dirname(backupFilePath), { recursive: true }, function (error) {
        if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, error.message));

        safe.fs.unlinkSync(backupFilePath); // remove any hardlink

        var fileStream = fs.createWriteStream(backupFilePath);

        // this pattern is required to ensure that the file got created before 'finish'
        fileStream.on('open', function () {
            sourceStream.pipe(fileStream);
        });

        fileStream.on('error', function (error) {
            debug('[%s] upload: out stream error.', backupFilePath, error);
            callback(new BoxError(BoxError.EXTERNAL_ERROR, error.message));
        });

        fileStream.on('finish', function () {
            const backupUid = parseInt(process.env.SUDO_UID, 10) || process.getuid(); // in test, upload() may or may not be called via sudo script

            if (hasChownSupportSync(apiConfig)) {
                if (!safe.fs.chownSync(backupFilePath, backupUid, backupUid)) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Unable to chown:' + safe.error.message));
                if (!safe.fs.chownSync(path.dirname(backupFilePath), backupUid, backupUid)) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Unable to chown:' + safe.error.message));
            }

            debug('upload %s: done.', backupFilePath);

            callback(null);
        });
    });
}

function download(apiConfig, sourceFilePath, callback) {
    assert.strictEqual(typeof apiConfig, 'object');
    assert.strictEqual(typeof sourceFilePath, 'string');
    assert.strictEqual(typeof callback, 'function');

    debug(`download: ${sourceFilePath}`);

    if (!safe.fs.existsSync(sourceFilePath)) return callback(new BoxError(BoxError.NOT_FOUND, `File not found: ${sourceFilePath}`));

    var fileStream = fs.createReadStream(sourceFilePath);
    callback(null, fileStream);
}

async function exists(apiConfig, sourceFilePath) {
    assert.strictEqual(typeof apiConfig, 'object');
    assert.strictEqual(typeof sourceFilePath, 'string');

    // do not use existsSync because it does not return EPERM etc
    if (!safe.fs.statSync(sourceFilePath)) {
        if (safe.error && safe.error.code === 'ENOENT') return false;
        if (safe.error) throw new BoxError(BoxError.EXTERNAL_ERROR, `Exists ${sourceFilePath}: ${safe.error.message}`);
    }

    return true;
}

function listDir(apiConfig, dir, batchSize, iteratorCallback, callback) {
    assert.strictEqual(typeof apiConfig, 'object');
    assert.strictEqual(typeof dir, 'string');
    assert.strictEqual(typeof batchSize, 'number');
    assert.strictEqual(typeof iteratorCallback, 'function');
    assert.strictEqual(typeof callback, 'function');

    var entries = [];
    var entryStream = readdirp(dir, { type: 'files', alwaysStat: true, lstat: true });
    entryStream.on('data', function (entryInfo) {
        if (entryInfo.stats.isSymbolicLink()) return;

        entries.push({ fullPath: entryInfo.fullPath });
        if (entries.length < batchSize) return;
        entryStream.pause();
        iteratorCallback(entries, function (error) {
            if (error) return callback(error);

            entries = [];
            entryStream.resume();
        });
    });
    entryStream.on('warn', function (error) {
        debug('listDir: warning ', error);
    });
    entryStream.on('end', function () {
        iteratorCallback(entries, callback);
    });
}

async function copy(apiConfig, oldFilePath, newFilePath, progressCallback) {
    assert.strictEqual(typeof apiConfig, 'object');
    assert.strictEqual(typeof oldFilePath, 'string');
    assert.strictEqual(typeof newFilePath, 'string');
    assert.strictEqual(typeof progressCallback, 'function');

    const [mkdirError] = await safe(fs.promises.mkdir(path.dirname(newFilePath), { recursive: true }));
    if (mkdirError) throw new BoxError(BoxError.EXTERNAL_ERROR, mkdirError.message);

    progressCallback({ message: `Copying ${oldFilePath} to ${newFilePath}` });

    let cpOptions = ((apiConfig.provider !== PROVIDER_MOUNTPOINT && apiConfig.provider !== PROVIDER_CIFS) || apiConfig.preserveAttributes) ? '-a' : '-dR';
    cpOptions += apiConfig.noHardlinks ? '' : 'l';  // this will hardlink backups saving space

    const [copyError] = await safe(shell.promises.spawn('copy', '/bin/cp', [ cpOptions, oldFilePath, newFilePath ], { }));
    if (copyError) throw new BoxError(BoxError.EXTERNAL_ERROR, copyError.message);
}

async function remove(apiConfig, filename) {
    assert.strictEqual(typeof apiConfig, 'object');
    assert.strictEqual(typeof filename, 'string');

    const stat = safe.fs.statSync(filename);
    if (!stat) return;

    if (stat.isFile()) {
        if (!safe.fs.unlinkSync(filename)) throw new BoxError(BoxError.EXTERNAL_ERROR, safe.error.message);
    } else if (stat.isDirectory()) {
        if (!safe.fs.rmSync(filename, { recursive: true })) throw new BoxError(BoxError.EXTERNAL_ERROR, safe.error.message);
    }
}

async function removeDir(apiConfig, pathPrefix, progressCallback) {
    assert.strictEqual(typeof apiConfig, 'object');
    assert.strictEqual(typeof pathPrefix, 'string');
    assert.strictEqual(typeof progressCallback, 'function');

    progressCallback({ message: `Removing directory ${pathPrefix}` });

    const [error] = await safe(shell.promises.spawn('removeDir', '/bin/rm', [ '-rf', pathPrefix ], { }));
    if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, error.message);
}

function validateBackupTarget(folder) {
    assert.strictEqual(typeof folder, 'string');

    if (path.normalize(folder) !== folder) return new BoxError(BoxError.BAD_FIELD, 'backupFolder/mountpoint must contain a normalized path');
    if (!path.isAbsolute(folder)) return new BoxError(BoxError.BAD_FIELD, 'backupFolder/mountpoint must be an absolute path');

    if (folder === '/') return new BoxError(BoxError.BAD_FIELD, 'backupFolder/mountpoint cannot be /');

    if (!folder.endsWith('/')) folder = folder + '/'; // ensure trailing slash for the prefix matching to work
    const PROTECTED_PREFIXES = [ '/boot/', '/usr/', '/bin/', '/lib/', '/root/', '/var/lib/', paths.baseDir() ];

    if (PROTECTED_PREFIXES.some(p => folder.startsWith(p))) return new BoxError(BoxError.BAD_FIELD, 'backupFolder path is protected');

    return null;
}

async function remount(apiConfig) {
    assert.strictEqual(typeof apiConfig, 'object');

    if (apiConfig.provider === PROVIDER_FILESYSTEM || apiConfig.provider === PROVIDER_MOUNTPOINT) return;

    await mounts.remount(mounts.mountObjectFromBackupConfig(apiConfig));
}

async function testConfig(apiConfig) {
    assert.strictEqual(typeof apiConfig, 'object');

    if ('noHardlinks' in apiConfig && typeof apiConfig.noHardlinks !== 'boolean') throw new BoxError(BoxError.BAD_FIELD, 'noHardlinks must be boolean');
    if ('chown' in apiConfig && typeof apiConfig.chown !== 'boolean') throw new BoxError(BoxError.BAD_FIELD, 'chown must be boolean');

    if (apiConfig.provider === PROVIDER_FILESYSTEM) {
        if (!apiConfig.backupFolder || typeof apiConfig.backupFolder !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'backupFolder must be non-empty string');
        const error = validateBackupTarget(apiConfig.backupFolder);
        if (error) throw error;
    } else { // xfs/cifs/ext4/nfs/mountpoint/sshfs
        if (apiConfig.provider === PROVIDER_MOUNTPOINT) {
            if (!apiConfig.mountPoint || typeof apiConfig.mountPoint !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'mountPoint must be non-empty string');
            const error = validateBackupTarget(apiConfig.mountPoint);
            if (error) throw error;
        }

        if (typeof apiConfig.prefix !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'prefix must be a string');
        if (apiConfig.prefix !== '') {
            if (path.isAbsolute(apiConfig.prefix)) throw new BoxError(BoxError.BAD_FIELD, 'prefix must be a relative path');
            if (path.normalize(apiConfig.prefix) !== apiConfig.prefix) throw new BoxError(BoxError.BAD_FIELD, 'prefix must contain a normalized relative path');
        }
    }

    if (apiConfig.provider === PROVIDER_MOUNTPOINT) {
        if (!safe.child_process.execSync(`mountpoint -q -- ${apiConfig.mountPoint}`)) throw new BoxError(BoxError.BAD_FIELD, `${apiConfig.mountPoint} is not mounted`);
    }

    const basePath = getBackupRootPath(apiConfig);
    const field = apiConfig.provider === PROVIDER_FILESYSTEM ? 'backupFolder' : 'mountPoint';

    if (!safe.fs.mkdirSync(path.join(basePath, 'snapshot'), { recursive: true }) && safe.error.code !== 'EEXIST') {
        if (safe.error && safe.error.code === 'EACCES') throw new BoxError(BoxError.BAD_FIELD, `Access denied. Create the directory and run "chown yellowtent:yellowtent ${basePath}" on the server`, { field });
        throw new BoxError(BoxError.BAD_FIELD, safe.error.message, { field });
    }

    if (!safe.fs.writeFileSync(path.join(basePath, 'cloudron-testfile'), 'testcontent')) {
        throw new BoxError(BoxError.BAD_FIELD, `Unable to create test file as 'yellowtent' user in ${basePath}: ${safe.error.message}. Check dir/mount permissions`, { field });
    }

    if (!safe.fs.unlinkSync(path.join(basePath, 'cloudron-testfile'))) {
        throw new BoxError(BoxError.BAD_FIELD, `Unable to remove test file as 'yellowtent' user in ${basePath}: ${safe.error.message}. Check dir/mount permissions`, { field });
    }
}

function removePrivateFields(apiConfig) {
    if (apiConfig.mountOptions && apiConfig.mountOptions.password) apiConfig.mountOptions.password = constants.SECRET_PLACEHOLDER;
    if (apiConfig.mountOptions && apiConfig.mountOptions.privateKey) apiConfig.mountOptions.privateKey = constants.SECRET_PLACEHOLDER;

    return apiConfig;
}

function injectPrivateFields(newConfig, currentConfig) {
    if (newConfig.mountOptions && currentConfig.mountOptions && newConfig.mountOptions.password === constants.SECRET_PLACEHOLDER) newConfig.mountOptions.password = currentConfig.mountOptions.password;
    if (newConfig.mountOptions && currentConfig.mountOptions && newConfig.mountOptions.privateKey === constants.SECRET_PLACEHOLDER) newConfig.mountOptions.privateKey = currentConfig.mountOptions.privateKey;
}