cloudron-box/src/storage/filesystem.js

'use strict';

exports = module.exports = {
    setup,
    teardown,
    cleanup,

    testConfig,
    removePrivateFields,
    injectPrivateFields,

    getAvailableSize,

    upload,
    download,

    copy,

    exists,
    listDir,

    remove,
    removeDir,
};

const assert = require('assert'),
    BoxError = require('../boxerror.js'),
    constants = require('../constants.js'),
    debug = require('debug')('box:storage/filesystem'),
    df = require('../df.js'),
    fs = require('fs'),
    mounts = require('../mounts.js'),
    path = require('path'),
    paths = require('../paths.js'),
    safe = require('safetydance'),
    shell = require('../shell.js')('filesystem');

async function getAvailableSize(apiConfig) {
    assert.strictEqual(typeof apiConfig, 'object');

    // note that df returns the disk size (as opposed to the apparent size)
    const [error, dfResult] = await safe(df.file(apiConfig.rootPath));
    if (error) throw new BoxError(BoxError.FS_ERROR, `Error when checking for disk space: ${error.message}`);

    return dfResult.available;
}

function hasChownSupportSync(apiConfig) {
    switch (apiConfig.provider) {
    case mounts.MOUNT_TYPE_NFS:
    case mounts.MOUNT_TYPE_EXT4:
    case mounts.MOUNT_TYPE_XFS:
    case mounts.MOUNT_TYPE_DISK:
    case mounts.MOUNT_TYPE_FILESYSTEM:
        return true;
    case mounts.MOUNT_TYPE_SSHFS:
        // sshfs can be mounted as root or normal user. when mounted as root, we have to chown since we remove backups as the yellowtent user
        // when mounted as non-root user, files are created as yellowtent user but they are still owned by the non-root user (thus del also works)
        return apiConfig.mountOptions.user === 'root';
    case mounts.MOUNT_TYPE_CIFS:
        return true;
    case mounts.MOUNT_TYPE_MOUNTPOINT:
        return apiConfig.chown;
    }
}

async function upload(apiConfig, backupFilePath) {
    assert.strictEqual(typeof apiConfig, 'object');
    assert.strictEqual(typeof backupFilePath, 'string');

    const [mkdirError] = await safe(fs.promises.mkdir(path.dirname(backupFilePath), { recursive: true }));
    if (mkdirError) throw new BoxError(BoxError.FS_ERROR, `Error creating directory ${backupFilePath}: ${mkdirError.message}`);

    await safe(fs.promises.unlink(backupFilePath)); // remove any hardlink

    return {
        stream: fs.createWriteStream(backupFilePath, { autoClose: true }),
        async finish() {
            const backupUid = parseInt(process.env.SUDO_UID, 10) || process.getuid(); // in test, upload() may or may not be called via sudo script
            if (hasChownSupportSync(apiConfig)) {
                if (!safe.fs.chownSync(backupFilePath, backupUid, backupUid)) throw new BoxError(BoxError.EXTERNAL_ERROR, `Unable to chown ${backupFilePath}: ${safe.error.message}`);
                if (!safe.fs.chownSync(path.dirname(backupFilePath), backupUid, backupUid)) throw new BoxError(BoxError.EXTERNAL_ERROR, `Unable to chown parentdir ${backupFilePath}: ${safe.error.message}`);
            }
        }
    };
}

async function download(apiConfig, sourceFilePath) {
    assert.strictEqual(typeof apiConfig, 'object');
    assert.strictEqual(typeof sourceFilePath, 'string');

    debug(`download: ${sourceFilePath}`);

    if (!safe.fs.existsSync(sourceFilePath)) throw new BoxError(BoxError.NOT_FOUND, `File not found: ${sourceFilePath}`);

    return fs.createReadStream(sourceFilePath);
}

async function exists(apiConfig, sourceFilePath) {
    assert.strictEqual(typeof apiConfig, 'object');
    assert.strictEqual(typeof sourceFilePath, 'string');

    // do not use existsSync because it does not return EPERM etc
    if (!safe.fs.statSync(sourceFilePath)) {
        if (safe.error && safe.error.code === 'ENOENT') return false;
        if (safe.error) throw new BoxError(BoxError.EXTERNAL_ERROR, `Exists ${sourceFilePath}: ${safe.error.message}`);
    }

    return true;
}

async function listDir(apiConfig, dir, batchSize, marker) {
    assert.strictEqual(typeof apiConfig, 'object');
    assert.strictEqual(typeof dir, 'string');
    assert.strictEqual(typeof batchSize, 'number');
    assert(typeof marker !== 'undefined');

    const stack = marker ? marker.stack : [dir];
    const fileStream = marker ? marker.fileStream : [];
    if (!marker) marker = { stack, fileStream };

    while (stack.length > 0) {
        const currentDir = stack.pop();
        const dirents = await fs.promises.readdir(currentDir, { withFileTypes: true });

        for (const dirent of dirents) {
            const fullPath = path.join(currentDir, dirent.name);

            if (dirent.isDirectory()) {
                stack.push(fullPath);
            } else if (dirent.isFile()) { // does not include symlink
                const stat = await fs.promises.lstat(fullPath);
                fileStream.push({ fullPath, size: stat.size });
            }
        }

        if (fileStream.length >= batchSize) return { entries: fileStream.splice(0, batchSize), marker }; // note: splice also modifies the array
    }

    if (fileStream.length === 0) return { entries: [], marker: null };

    return { entries: fileStream.splice(0, batchSize), marker };  // note: splice also modifies the array
}

async function copy(apiConfig, oldFilePath, newFilePath, progressCallback) {
    assert.strictEqual(typeof apiConfig, 'object');
    assert.strictEqual(typeof oldFilePath, 'string');
    assert.strictEqual(typeof newFilePath, 'string');
    assert.strictEqual(typeof progressCallback, 'function');

    const [mkdirError] = await safe(fs.promises.mkdir(path.dirname(newFilePath), { recursive: true }));
    if (mkdirError) throw new BoxError(BoxError.EXTERNAL_ERROR, mkdirError.message);

    progressCallback({ message: `Copying ${oldFilePath} to ${newFilePath}` });

    let cpOptions = ((apiConfig.provider !== mounts.MOUNT_TYPE_MOUNTPOINT && apiConfig.provider !== mounts.MOUNT_TYPE_CIFS) || apiConfig.preserveAttributes) ? '-a' : '-dR';
    cpOptions += apiConfig.noHardlinks ? '' : 'l';  // this will hardlink backups saving space

    if (apiConfig.provider === mounts.MOUNT_TYPE_SSHFS) {
        const identityFilePath = path.join(paths.SSHFS_KEYS_DIR, `id_rsa_${apiConfig.mountOptions.host}`);

        const sshOptions = [ '-o', '"StrictHostKeyChecking no"', '-i', identityFilePath, '-p', apiConfig.mountOptions.port, `${apiConfig.mountOptions.user}@${apiConfig.mountOptions.host}` ];
        const sshArgs = sshOptions.concat([ 'cp', cpOptions, oldFilePath.replace('/mnt/cloudronbackup/', ''), newFilePath.replace('/mnt/cloudronbackup/', '') ]);
        const [remoteCopyError] = await safe(shell.spawn('ssh', sshArgs, { shell: true }));
        if (!remoteCopyError) return;
        if (remoteCopyError.code === 255) throw new BoxError(BoxError.EXTERNAL_ERROR, `SSH connection error: ${remoteCopyError.message}`); // do not attempt fallback copy for ssh errors
        debug('SSH remote copy failed, trying sshfs copy'); // this can happen for sshfs mounted windows server
    }

    const [copyError] = await safe(shell.spawn('cp', [ cpOptions, oldFilePath, newFilePath ], {}));
    if (copyError) throw new BoxError(BoxError.EXTERNAL_ERROR, copyError.message);
}

async function remove(apiConfig, filename) {
    assert.strictEqual(typeof apiConfig, 'object');
    assert.strictEqual(typeof filename, 'string');

    const stat = safe.fs.statSync(filename);
    if (!stat) return;

    if (stat.isFile()) {
        if (!safe.fs.unlinkSync(filename)) throw new BoxError(BoxError.EXTERNAL_ERROR, safe.error.message);
    } else if (stat.isDirectory()) {
        if (!safe.fs.rmdirSync(filename, { recursive: false })) throw new BoxError(BoxError.EXTERNAL_ERROR, safe.error.message);
    }
}

async function removeDir(apiConfig, pathPrefix, progressCallback) {
    assert.strictEqual(typeof apiConfig, 'object');
    assert.strictEqual(typeof pathPrefix, 'string');
    assert.strictEqual(typeof progressCallback, 'function');

    progressCallback({ message: `Removing directory ${pathPrefix}` });

    if (apiConfig.provider === mounts.MOUNT_TYPE_SSHFS) {
        const identityFilePath = path.join(paths.SSHFS_KEYS_DIR, `id_rsa_${apiConfig.mountOptions.host}`);

        const sshOptions = [ '-o', '"StrictHostKeyChecking no"', '-i', identityFilePath, '-p', apiConfig.mountOptions.port, `${apiConfig.mountOptions.user}@${apiConfig.mountOptions.host}` ];
        const sshArgs = sshOptions.concat([ 'rm', '-rf', pathPrefix.replace('/mnt/cloudronbackup/', '') ]);
        const [remoteRmError] = await safe(shell.spawn('ssh', sshArgs, { shell: true }));
        if (!remoteRmError) return;
        if (remoteRmError.code === 255) throw new BoxError(BoxError.EXTERNAL_ERROR, `SSH connection error: ${remoteRmError.message}`); // do not attempt fallback copy for ssh errors
        debug('SSH remote rm failed, trying sshfs rm'); // this can happen for sshfs mounted windows server
    }

    const [error] = await safe(shell.spawn('rm', [ '-rf', pathPrefix ], {}));
    if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, error.message);
}

function validateDestPath(folder) {
    assert.strictEqual(typeof folder, 'string');

    if (path.normalize(folder) !== folder) return new BoxError(BoxError.BAD_FIELD, 'backupFolder/mountpoint must contain a normalized path');
    if (!path.isAbsolute(folder)) return new BoxError(BoxError.BAD_FIELD, 'backupFolder/mountpoint must be an absolute path');

    if (folder === '/') return new BoxError(BoxError.BAD_FIELD, 'backupFolder/mountpoint cannot be /');

    if (!folder.endsWith('/')) folder = folder + '/'; // ensure trailing slash for the prefix matching to work
    const PROTECTED_PREFIXES = [ '/boot/', '/usr/', '/bin/', '/lib/', '/root/', '/var/lib/', paths.baseDir() ];

    if (PROTECTED_PREFIXES.some(p => folder.startsWith(p))) return new BoxError(BoxError.BAD_FIELD, 'backupFolder path is protected');

    return null;
}

async function cleanup(apiConfig, progressCallback) {
    assert.strictEqual(typeof apiConfig, 'object');
    assert.strictEqual(typeof progressCallback, 'function');
}

async function setupManagedMount(apiConfig, hostPath) {
    assert.strictEqual(typeof apiConfig, 'object');
    assert.strictEqual(typeof hostPath, 'string');
    assert(mounts.isManagedProvider(apiConfig.provider));

    if (!apiConfig.mountOptions || typeof apiConfig.mountOptions !== 'object') throw new BoxError(BoxError.BAD_FIELD, 'mountOptions must be an object');

    const error = mounts.validateMountOptions(apiConfig.provider, apiConfig.mountOptions);
    if (error) throw error;

    debug(`setupManagedMount: setting up mount at ${hostPath} with ${apiConfig.provider}`);

    const newMount = {
        name: path.basename(hostPath),
        hostPath,
        mountType: apiConfig.provider,
        mountOptions: apiConfig.mountOptions
    };

    await mounts.tryAddMount(newMount, { timeout: 10 }); // 10 seconds

    return newMount;
}

async function setup(apiConfig) {
    assert.strictEqual(typeof apiConfig, 'object');

    debug('setup: removing old storage configuration');
    if (!mounts.isManagedProvider(apiConfig.provider)) return;
    const oldMountObject = {
        name: 'backup',
        hostPath: paths.MANAGED_BACKUP_MOUNT_DIR,
        mountType: apiConfig.provider,
        mountOptions: apiConfig.mountOptions // must have already been validated
    };

    await safe(mounts.removeMount(oldMountObject), { debug }); // ignore error

    debug('setup: setting up new storage configuration');
    await setupManagedMount(apiConfig, paths.MANAGED_BACKUP_MOUNT_DIR);
}

async function teardown(apiConfig) {
    assert.strictEqual(typeof apiConfig, 'object');

    if (!mounts.isManagedProvider(apiConfig.provider)) return;
    const mountObject = {
        name: 'backup',
        hostPath: paths.MANAGED_BACKUP_MOUNT_DIR,
        mountType: apiConfig.provider,
        mountOptions: apiConfig.mountOptions
    };

    await safe(mounts.removeMount(mountObject), { debug }); // ignore error
}

async function testConfig(apiConfig) {
    assert.strictEqual(typeof apiConfig, 'object');

    if ('noHardlinks' in apiConfig && typeof apiConfig.noHardlinks !== 'boolean') throw new BoxError(BoxError.BAD_FIELD, 'noHardlinks must be boolean');
    if ('chown' in apiConfig && typeof apiConfig.chown !== 'boolean') throw new BoxError(BoxError.BAD_FIELD, 'chown must be boolean');
    if ('preserveAttributes' in apiConfig && typeof apiConfig.preserveAttributes !== 'boolean') throw new BoxError(BoxError.BAD_FIELD, 'preserveAttributes must be boolean');

    let rootPath, testMountObject;
    if (apiConfig.provider === mounts.MOUNT_TYPE_FILESYSTEM) {
        if (!apiConfig.backupFolder || typeof apiConfig.backupFolder !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'backupFolder must be non-empty string');
        const error = validateDestPath(apiConfig.backupFolder);
        if (error) throw error;
        rootPath = apiConfig.backupFolder;
    } else {
        if (typeof apiConfig.prefix !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'prefix must be a string');
        if (apiConfig.prefix !== '') {
            if (path.isAbsolute(apiConfig.prefix)) throw new BoxError(BoxError.BAD_FIELD, 'prefix must be a relative path');
            if (path.normalize(apiConfig.prefix) !== apiConfig.prefix) throw new BoxError(BoxError.BAD_FIELD, 'prefix must contain a normalized relative path');
        }

        if (mounts.isManagedProvider(apiConfig.provider)) {
            testMountObject = await setupManagedMount(apiConfig, '/mnt/backup-storage-validation'); // this validates mountOptions
            rootPath = path.join('/mnt/backup-storage-validation', apiConfig.prefix);
        } else if (apiConfig.provider === mounts.MOUNT_TYPE_MOUNTPOINT) {
            if (!apiConfig.mountPoint || typeof apiConfig.mountPoint !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'mountPoint must be non-empty string');
            const error = validateDestPath(apiConfig.mountPoint);
            if (error) throw error;

            const [mountError] = await safe(shell.spawn('mountpoint', ['-q', '--', apiConfig.mountPoint], { timeout: 5000 }));
            if (mountError) throw new BoxError(BoxError.BAD_FIELD, `${apiConfig.mountPoint} is not mounted: ${mountError.message}`);
            rootPath = path.join(apiConfig.mountPoint, apiConfig.prefix);
        } else {
            throw new BoxError(BoxError.BAD_FIELD, `Unknown provider: ${apiConfig.provider}`);
        }
    }

    if (!safe.fs.mkdirSync(path.join(rootPath, 'snapshot'), { recursive: true }) && safe.error.code !== 'EEXIST') {
        if (safe.error && safe.error.code === 'EACCES') throw new BoxError(BoxError.BAD_FIELD, `Access denied. Create ${rootPath}/snapshot and run "chown yellowtent:yellowtent ${rootPath}" on the server`);
        throw new BoxError(BoxError.BAD_FIELD, safe.error.message);
    }

    if (!safe.fs.writeFileSync(path.join(rootPath, 'cloudron-testfile'), 'testcontent')) {
        throw new BoxError(BoxError.BAD_FIELD, `Unable to create test file as 'yellowtent' user in ${rootPath}: ${safe.error.message}. Check dir/mount permissions`);
    }

    if (!safe.fs.unlinkSync(path.join(rootPath, 'cloudron-testfile'))) {
        throw new BoxError(BoxError.BAD_FIELD, `Unable to remove test file as 'yellowtent' user in ${rootPath}: ${safe.error.message}. Check dir/mount permissions`);
    }

    if (testMountObject) await mounts.removeMount(testMountObject);
}

function removePrivateFields(apiConfig) {
    if (apiConfig.mountOptions && apiConfig.mountOptions.password) apiConfig.mountOptions.password = constants.SECRET_PLACEHOLDER;
    if (apiConfig.mountOptions && apiConfig.mountOptions.privateKey) apiConfig.mountOptions.privateKey = constants.SECRET_PLACEHOLDER;

    return apiConfig;
}

function injectPrivateFields(newConfig, currentConfig) {
    if (newConfig.mountOptions && currentConfig.mountOptions && newConfig.mountOptions.password === constants.SECRET_PLACEHOLDER) newConfig.mountOptions.password = currentConfig.mountOptions.password;
    if (newConfig.mountOptions && currentConfig.mountOptions && newConfig.mountOptions.privateKey === constants.SECRET_PLACEHOLDER) newConfig.mountOptions.privateKey = currentConfig.mountOptions.privateKey;
}