cloudron-box/src/system.js

'use strict';

exports = module.exports = {
    reboot,
    getInfo,
    getUbuntuVersion,
    getSwaps,
    checkDiskSpace,
    getMemory,
    getDiskUsage,
    updateDiskUsage,
    startUpdateDiskUsage,
    getLogs,
    getBlockDevices,
    runSystemChecks,
    getProvider,
    getCpus,

    // exported for testing
    _getFilesystems: getFilesystems
};

const apps = require('./apps.js'),
    assert = require('assert'),
    backups = require('./backups.js'),
    BoxError = require('./boxerror.js'),
    debug = require('debug')('box:system'),
    df = require('./df.js'),
    docker = require('./docker.js'),
    eventlog = require('./eventlog.js'),
    fs = require('fs'),
    logs = require('./logs.js'),
    notifications = require('./notifications.js'),
    os = require('os'),
    path = require('path'),
    paths = require('./paths.js'),
    safe = require('safetydance'),
    shell = require('./shell.js')('system'),
    tasks = require('./tasks.js'),
    volumes = require('./volumes.js');

const DU_CMD = path.join(__dirname, 'scripts/du.sh');
const HDPARM_CMD = path.join(__dirname, 'scripts/hdparm.sh');
const REBOOT_CMD = path.join(__dirname, 'scripts/reboot.sh');

async function du(file) {
    assert.strictEqual(typeof file, 'string');

    const [error, stdoutResult] = await safe(shell.promises.sudo([ DU_CMD, file ], { captureStdout: true }));
    if (error) throw new BoxError(BoxError.FS_ERROR, error);

    return parseInt(stdoutResult.trim(), 10);
}

async function hdparm(file) {
    assert.strictEqual(typeof file, 'string');

    const [error, stdoutResult] = await safe(shell.promises.sudo([ HDPARM_CMD, file ], { captureStdout: true }));
    if (error) throw new BoxError(BoxError.FS_ERROR, error);

    const lines = stdoutResult.split('\n');

    if (lines.length != 4) return -1;
    if (lines[2].split('=').length !== 2) return -1;

    const speed = lines[2].split('=')[1].slice(0, 'MB/sec'.length).trim();

    return Number(speed);
}

async function getSwaps() {
    const [error, stdout] = await safe(shell.spawn('swapon', ['--noheadings', '--raw', '--bytes', '--show=type,size,used,name'], { encoding: 'utf8' }));
    if (error) return {};
    const output = stdout.trim();
    if (!output) return {}; // no swaps

    const swaps = {};
    for (const line of output.split('\n')) {
        const parts = line.split(' ', 4);
        const name = parts[3];
        swaps[name] = {
            name: parts[3],
            type: parts[0], // partition or file
            size: parseInt(parts[1]),
            used: parseInt(parts[2]),
        };
    }

    return swaps;
}

// this gets information based on mounted filesystems
async function getFilesystems() {
    const FS_TYPES = [ 'ext4', 'xfs', 'cifs', 'nfs', 'fuse.sshfs' ]; // we don't show size of contents in untracked disk types

    const [dfError, dfEntries] = await safe(df.filesystems());
    if (dfError) throw new BoxError(BoxError.FS_ERROR, `Error running df: ${dfError.message}`);

    const filesystems = {}; // by file system
    let rootDisk;

    for (const dfEntry of dfEntries) {
        if (!FS_TYPES.includes(dfEntry.type)) continue;
        if (dfEntry.mountpoint === '/') rootDisk = dfEntry;
        filesystems[dfEntry.filesystem] = {
            filesystem: dfEntry.filesystem,
            type: dfEntry.type,
            size: dfEntry.size,
            used: dfEntry.used,
            available: dfEntry.available,
            capacity: dfEntry.capacity,
            mountpoint: dfEntry.mountpoint,
            contents: [] // filled below
        };
    }

    const standardPaths = [
        { type: 'standard', id: 'platformdata', path: paths.PLATFORM_DATA_DIR },
        { type: 'standard', id: 'boxdata',      path: paths.BOX_DATA_DIR },
        { type: 'standard', id: 'maildata',     path: paths.MAIL_DATA_DIR },
    ];

    for (const stdPath of standardPaths) {
        const [dfError, diskInfo] = await safe(df.file(stdPath.path));
        if (dfError) throw new BoxError(BoxError.FS_ERROR, `Error getting std path: ${dfError.message}`);
        filesystems[diskInfo.filesystem].contents.push(stdPath);
    }

    const backupConfig = await backups.getConfig();
    if (backupConfig.provider === 'filesystem') {
        const [, dfResult] = await safe(df.file(backupConfig.backupFolder));
        const filesystem = dfResult?.filesystem || rootDisk.filesystem;
        if (filesystems[filesystem]) filesystems[filesystem].contents.push({ type: 'standard', id: 'cloudron-backup', path: backupConfig.backupFolder });
    }

    // often the default backup dir is not cleaned up
    if (backupConfig.provider !== 'filesystem' || backupConfig.backupFolder !== paths.DEFAULT_BACKUP_DIR) {
        const [, dfResult] = await safe(df.file(paths.DEFAULT_BACKUP_DIR));
        const filesystem = dfResult?.filesystem || rootDisk.filesystem;
        if (filesystems[filesystem]) filesystems[filesystem].contents.push({ type: 'cloudron-backup-default', id: 'cloudron-backup-default', path: paths.DEFAULT_BACKUP_DIR });
    }

    const [dockerError, dockerInfo] = await safe(docker.info());
    if (!dockerError) {
        const [, dfResult] = await safe(df.file(dockerInfo.DockerRootDir));
        const filesystem = dfResult?.filesystem || rootDisk.filesystem;
        if (filesystems[filesystem]) {
            filesystems[filesystem].contents.push({ type: 'standard', id: 'docker', path: dockerInfo.DockerRootDir });
            filesystems[filesystem].contents.push({ type: 'standard', id: 'docker-volumes', path: dockerInfo.DockerRootDir });
        }
    }

    for (const volume of await volumes.list()) {
        const [, dfResult] = await safe(df.file(volume.hostPath));
        const filesystem = dfResult?.filesystem || rootDisk.filesystem;
        if (filesystems[filesystem]) filesystems[filesystem].contents.push({ type: 'volume', id: volume.id, path: volume.hostPath });
    }

    for (const app of await apps.list()) {
        if (!app.manifest.addons?.localstorage) continue;

        const dataDir = await apps.getStorageDir(app);
        if (dataDir === null) continue;
        const [, dfResult] = await safe(df.file(dataDir));
        const filesystem = dfResult?.filesystem || rootDisk.filesystem;
        if (filesystems[filesystem]) filesystems[filesystem].contents.push({ type: 'app', id: app.id, path: dataDir });
    }

    const swaps = await getSwaps();
    for (const k in swaps) {
        const swap = swaps[k];
        if (swap.type !== 'file') continue;

        const [, dfResult] = await safe(df.file(swap.name));
        filesystems[dfResult?.filesystem || rootDisk.filesystem].contents.push({ type: 'swap', id: swap.name, path: swap.name });
    }

    return filesystems;
}

async function checkDiskSpace() {
    debug('checkDiskSpace: checking disk space');

    const filesystems = await getFilesystems();

    let markdownMessage = '';

    for (const fsPath in filesystems) {
        const filesystem = filesystems[fsPath];
        if (filesystem.contents.length === 0) continue; // ignore if nothing interesting here

        if (filesystem.available <= (1.25 * 1024 * 1024 * 1024)) { // 1.5G
            markdownMessage += `* ${filesystem.filesystem} is at ${filesystem.capacity*100}% capacity.\n`;
        }
    }

    debug(`checkDiskSpace: disk space checked. out of space: ${markdownMessage || 'no'}`);

    if (markdownMessage) {
        const finalMessage = `One or more file systems are running out of space. Please increase the disk size at the earliest.\n\n${markdownMessage}`;
        await notifications.pin(notifications.TYPE_DISK_SPACE, 'Server is running out of disk space', finalMessage, {});
    } else {
        await notifications.unpin(notifications.TYPE_DISK_SPACE, {});
    }
}

async function getSwapSize() {
    const swaps = await getSwaps();
    return Object.keys(swaps).map(n => swaps[n].size).reduce((acc, cur) => acc + cur, 0);
}

async function getMemory() {
    return {
        memory: os.totalmem(),
        swap: await getSwapSize()
    };
}

async function getDiskUsage() {
    const cache = safe.JSON.parse(safe.fs.readFileSync(paths.DISK_USAGE_CACHE_FILE, 'utf8'));
    if (cache?.disks) {
        cache.filesystems = cache.disks; // legacy cache file had "disks"
        delete cache.disks;
    }
    return cache;
}

async function updateDiskUsage(progressCallback) {
    assert.strictEqual(typeof progressCallback, 'function');

    const filesystems = await getFilesystems();
    const now = Date.now();

    let percent = 1;

    const dockerDf = await docker.df();
    const excludePaths = (safe.fs.readFileSync(paths.DISK_USAGE_EXCLUDE_FILE, 'utf8') || '').split('\n');

    const fsCount = Object.keys(filesystems).length;
    for (const fsPath in filesystems) {
        const filesystem = filesystems[fsPath];

        if (filesystem.type === 'ext4' || filesystem.type === 'xfs') { // hdparm only works with block devices
            const [speedError, speed] = await safe(hdparm(fsPath));
            if (speedError) progressCallback({ message: `hdparm error: ${speedError.message}`});
            filesystem.speed = speedError ? -1 : speed;
        } else {
            filesystem.speed = -1;
        }

        percent += (100/fsCount);
        progressCallback({ percent, message: `Checking contents of ${fsPath}`});

        for (const content of filesystem.contents) {
            progressCallback({ message: `Checking du of ${content.id} ${content.path}`});
            if (content.id === 'docker') {
                content.usage = dockerDf.LayersSize;
            } else if (content.id === 'docker-volumes') {
                content.usage = dockerDf.Volumes.map((v) => v.UsageData.Size).reduce((a,b) => a + b, 0);
            } else if (excludePaths.includes(fsPath)) {
                debug(`updateDiskUsage: skipping since path ${fsPath} is excluded`);
                content.usage = 0;
            } else {
                const [error, usage] = await safe(du(content.path));
                if (error) progressCallback({ message: `du error: ${error.message}`}); // can happen if app is installing etc
                content.usage = usage || 0;
            }
            progressCallback({ message: `du of ${JSON.stringify(content)}: ${content.usage}`});
        }
    }

    if (!safe.fs.writeFileSync(paths.DISK_USAGE_CACHE_FILE, JSON.stringify({ ts: now, filesystems }, null, 4), 'utf8')) throw new BoxError(BoxError.FS_ERROR, `Could not write du cache file: ${safe.error.message}`);

    return filesystems;
}

async function reboot() {
    await notifications.unpin(notifications.TYPE_REBOOT, {});

    const [error] = await safe(shell.promises.sudo([ REBOOT_CMD ], {}));
    if (error) debug('reboot: could not reboot. %o', error);
}

async function getInfo() {
    // https://serverfault.com/questions/92932/how-does-ubuntu-keep-track-of-the-system-restart-required-flag-in-motd
    const rebootRequired = fs.existsSync('/var/run/reboot-required');
    const uptime = safe.fs.readFileSync('/proc/uptime', 'utf8');
    const uptimeSecs = parseInt(uptime.split(' ')[0], 10);

    const sysVendor = safe.fs.readFileSync('/sys/devices/virtual/dmi/id/sys_vendor', 'utf8');
    const productName = safe.fs.readFileSync('/sys/devices/virtual/dmi/id/product_name', 'utf8');

    const activationTime = (await eventlog.getActivationEvent())?.creationTime || null;

    return {
        sysVendor: sysVendor.trim(),
        productName: productName.trim(),
        uptimeSecs,
        rebootRequired,
        activationTime
    };
}

async function startUpdateDiskUsage() {
    const taskId = await tasks.add(tasks.TASK_UPDATE_DISK_USAGE, []);
    tasks.startTask(taskId, {});
    return taskId;
}

async function getLogs(unit, options) {
    assert.strictEqual(typeof unit, 'string');
    assert(options && typeof options === 'object');

    if (unit !== 'box') throw new BoxError(BoxError.BAD_FIELD, `No such unit '${unit}'`);

    const logFile = path.join(paths.LOG_DIR, 'box.log');
    const cp = logs.tail([logFile], { lines: options.lines, follow: options.follow });

    const logStream = new logs.LogStream({ format: options.format || 'json', source: unit });
    logStream.on('close', () => cp.terminate()); // the caller has to call destroy() on logStream. destroy() of Transform emits 'close'

    cp.stdout.pipe(logStream);

    return logStream;
}

// this gets block devices as opposed to mounted filesystems. this is used for configuring backups and volumes in the frontend
async function getBlockDevices() {
    const result = await shell.spawn('lsblk', ['--paths', '--json', '--list', '--fs', '--output', '+rota'], { encoding: 'utf8' });
    const info = safe.JSON.parse(result);
    if (!info) throw new BoxError(BoxError.INTERNAL_ERROR, `failed to parse lsblk: ${safe.error.message}`);

    const devices = info.blockdevices.filter(d => d.fstype === 'ext4' || d.fstype === 'xfs');

    debug(`getBlockDevices: Found ${devices.length} devices. ${devices.map(d => d.name).join(', ')}`);

    return devices.map(function (d) {
        return {
            path: d.name,
            size: d.fsavail || 0,
            type: d.fstype,
            uuid: d.uuid,
            rota: d.rota, // false (ssd) true (hdd)
            mountpoint: d.mountpoints ? d.mountpoints.pop() : d.mountpoint // we only support one mountpoint here old lsblk only exposed one via .mountpoint
        };
    });
}

async function checkRebootRequired() {
    const { rebootRequired } = await getInfo();
    if (rebootRequired) {
        await notifications.pin(notifications.TYPE_REBOOT, 'Reboot Required', 'To finish ubuntu security updates, a reboot is necessary.', {});
    } else {
        await notifications.unpin(notifications.TYPE_REBOOT, {});
    }
}

async function getUbuntuVersion() {
    const release = safe.fs.readFileSync('/etc/lsb-release', 'utf-8');
    if (release === null) throw new BoxError(BoxError.FS_ERROR, safe.error.message);
    return release.match(/DISTRIB_DESCRIPTION="(.*)"/)[1];
}

async function checkUbuntuVersion() {
    const isBionic = fs.readFileSync('/etc/lsb-release', 'utf-8').includes('18.04');
    if (!isBionic) return;

    await notifications.pin(notifications.TYPE_UPDATE_UBUNTU, 'Ubuntu upgrade required', 'Ubuntu 18.04 has reached end of life and will not receive security and maintenance updates. Please follow https://docs.cloudron.io/guides/upgrade-ubuntu-20/ to upgrade to Ubuntu 20 at the earliest.', {});
}

async function runSystemChecks() {
    debug('runSystemChecks: checking status');

    const checks = [
        checkRebootRequired(),
        checkUbuntuVersion()
    ];

    await Promise.allSettled(checks);
}

function getProvider() {
    const provider = safe.fs.readFileSync(paths.PROVIDER_FILE, 'utf8');
    return provider ? provider.trim() : 'generic';
}

async function getCpus() {
    return os.cpus();
}