'use strict'; exports = module.exports = { reboot, getInfo, getUbuntuVersion, getSwaps, checkDiskSpace, getMemory, getDiskUsage, updateDiskUsage, startUpdateDiskUsage, getLogs, getBlockDevices, runSystemChecks, getProvider, getCpus, // exported for testing _getFilesystems: getFilesystems }; const apps = require('./apps.js'), assert = require('assert'), backups = require('./backups.js'), BoxError = require('./boxerror.js'), debug = require('debug')('box:system'), df = require('./df.js'), docker = require('./docker.js'), eventlog = require('./eventlog.js'), fs = require('fs'), logs = require('./logs.js'), notifications = require('./notifications.js'), os = require('os'), path = require('path'), paths = require('./paths.js'), safe = require('safetydance'), shell = require('./shell.js')('system'), tasks = require('./tasks.js'), volumes = require('./volumes.js'); const DU_CMD = path.join(__dirname, 'scripts/du.sh'); const HDPARM_CMD = path.join(__dirname, 'scripts/hdparm.sh'); const REBOOT_CMD = path.join(__dirname, 'scripts/reboot.sh'); async function du(file) { assert.strictEqual(typeof file, 'string'); const [error, stdoutResult] = await safe(shell.promises.sudo([ DU_CMD, file ], { captureStdout: true })); if (error) throw new BoxError(BoxError.FS_ERROR, error); return parseInt(stdoutResult.trim(), 10); } async function hdparm(file) { assert.strictEqual(typeof file, 'string'); const [error, stdoutResult] = await safe(shell.promises.sudo([ HDPARM_CMD, file ], { captureStdout: true })); if (error) throw new BoxError(BoxError.FS_ERROR, error); const lines = stdoutResult.split('\n'); if (lines.length != 4) return -1; if (lines[2].split('=').length !== 2) return -1; const speed = lines[2].split('=')[1].slice(0, 'MB/sec'.length).trim(); return Number(speed); } async function getSwaps() { const [error, stdout] = await safe(shell.spawn('swapon', ['--noheadings', '--raw', '--bytes', '--show=type,size,used,name'], { encoding: 'utf8' })); if (error) return {}; const output = stdout.trim(); if (!output) return {}; // no swaps const swaps = {}; for (const line of output.split('\n')) { const parts = line.split(' ', 4); const name = parts[3]; swaps[name] = { name: parts[3], type: parts[0], // partition or file size: parseInt(parts[1]), used: parseInt(parts[2]), }; } return swaps; } // this gets information based on mounted filesystems async function getFilesystems() { const [dfError, dfEntries] = await safe(df.filesystems()); if (dfError) throw new BoxError(BoxError.FS_ERROR, `Error running df: ${dfError.message}`); const filesystems = {}; // by file system let rootDisk; const FS_TYPES = [ 'ext4', 'xfs', 'cifs', 'nfs', 'fuse.sshfs' ]; // we don't show size of contents in untracked disk types for (const dfEntry of dfEntries) { if (!FS_TYPES.includes(dfEntry.type)) continue; if (dfEntry.mountpoint === '/') rootDisk = dfEntry; filesystems[dfEntry.filesystem] = { filesystem: dfEntry.filesystem, type: dfEntry.type, size: dfEntry.size, used: dfEntry.used, available: dfEntry.available, capacity: dfEntry.capacity, mountpoint: dfEntry.mountpoint, contents: [] // filled below }; } const standardPaths = [ { type: 'standard', id: 'platformdata', path: paths.PLATFORM_DATA_DIR }, { type: 'standard', id: 'boxdata', path: paths.BOX_DATA_DIR }, { type: 'standard', id: 'maildata', path: paths.MAIL_DATA_DIR }, ]; for (const stdPath of standardPaths) { const [dfError, diskInfo] = await safe(df.file(stdPath.path)); if (dfError) throw new BoxError(BoxError.FS_ERROR, `Error getting std path: ${dfError.message}`); filesystems[diskInfo.filesystem].contents.push(stdPath); } const backupConfig = await backups.getConfig(); if (backupConfig.provider === 'filesystem') { const [, dfResult] = await safe(df.file(backupConfig.backupFolder)); const filesystem = dfResult?.filesystem || rootDisk.filesystem; if (filesystems[filesystem]) filesystems[filesystem].contents.push({ type: 'standard', id: 'cloudron-backup', path: backupConfig.backupFolder }); } // often the default backup dir is not cleaned up if (backupConfig.provider !== 'filesystem' || backupConfig.backupFolder !== paths.DEFAULT_BACKUP_DIR) { const [, dfResult] = await safe(df.file(paths.DEFAULT_BACKUP_DIR)); const filesystem = dfResult?.filesystem || rootDisk.filesystem; if (filesystems[filesystem]) filesystems[filesystem].contents.push({ type: 'cloudron-backup-default', id: 'cloudron-backup-default', path: paths.DEFAULT_BACKUP_DIR }); } const [dockerError, dockerInfo] = await safe(docker.info()); if (!dockerError) { const [, dfResult] = await safe(df.file(dockerInfo.DockerRootDir)); const filesystem = dfResult?.filesystem || rootDisk.filesystem; if (filesystems[filesystem]) { filesystems[filesystem].contents.push({ type: 'standard', id: 'docker', path: dockerInfo.DockerRootDir }); filesystems[filesystem].contents.push({ type: 'standard', id: 'docker-volumes', path: dockerInfo.DockerRootDir }); } } for (const volume of await volumes.list()) { const [, dfResult] = await safe(df.file(volume.hostPath)); const filesystem = dfResult?.filesystem || rootDisk.filesystem; if (filesystems[filesystem]) filesystems[filesystem].contents.push({ type: 'volume', id: volume.id, path: volume.hostPath }); } for (const app of await apps.list()) { if (!app.manifest.addons?.localstorage) continue; const dataDir = await apps.getStorageDir(app); if (dataDir === null) continue; const [, dfResult] = await safe(df.file(dataDir)); const filesystem = dfResult?.filesystem || rootDisk.filesystem; if (filesystems[filesystem]) filesystems[filesystem].contents.push({ type: 'app', id: app.id, path: dataDir }); } const swaps = await getSwaps(); for (const k in swaps) { const swap = swaps[k]; if (swap.type !== 'file') continue; const [, dfResult] = await safe(df.file(swap.name)); filesystems[dfResult?.filesystem || rootDisk.filesystem].contents.push({ type: 'swap', id: swap.name, path: swap.name }); } return filesystems; } async function checkDiskSpace() { debug('checkDiskSpace: checking disk space'); const filesystems = await getFilesystems(); let markdownMessage = ''; for (const fsPath in filesystems) { const filesystem = filesystems[fsPath]; if (filesystem.contents.length === 0) continue; // ignore if nothing interesting here if (filesystem.available <= (1.25 * 1024 * 1024 * 1024)) { // 1.5G markdownMessage += `* ${filesystem.filesystem} is at ${filesystem.capacity*100}% capacity.\n`; } } debug(`checkDiskSpace: disk space checked. out of space: ${markdownMessage || 'no'}`); if (markdownMessage) { const finalMessage = `One or more file systems are running out of space. Please increase the disk size at the earliest.\n\n${markdownMessage}`; await notifications.alert(notifications.ALERT_DISK_SPACE, 'Server is running out of disk space', finalMessage, { persist: true }); } else { await notifications.clearAlert(notifications.ALERT_DISK_SPACE, 'Server is running out of disk space'); } } async function getSwapSize() { const swaps = await getSwaps(); return Object.keys(swaps).map(n => swaps[n].size).reduce((acc, cur) => acc + cur, 0); } async function getMemory() { return { memory: os.totalmem(), swap: await getSwapSize() }; } async function getDiskUsage() { const cache = safe.JSON.parse(safe.fs.readFileSync(paths.DISK_USAGE_FILE, 'utf8')); if (cache?.disks) { cache.filesystems = cache.disks; // legacy cache file had "disks" delete cache.disks; } return cache; } async function updateDiskUsage(progressCallback) { assert.strictEqual(typeof progressCallback, 'function'); const filesystems = await getFilesystems(); const now = Date.now(); let percent = 1; for (const fsPath in filesystems) { const filesystem = filesystems[fsPath]; if (filesystem.type === 'ext4' || filesystem.type === 'xfs') { const [speedError, speed] = await safe(hdparm(filesystem)); if (speedError) progressCallback({ message: `hdparm error: ${speedError.message}`}); filesystem.speed = speedError ? -1 : speed; } else { filesystem.speed = -1; } percent += (100/filesystems.length); progressCallback({ percent, message: `Checking contents of ${filesystem}`}); const dockerDf = await docker.df(); for (const content of filesystem.contents) { progressCallback({ message: `Checking du of ${JSON.stringify(content)}`}); if (content.id === 'docker') { content.usage = dockerDf.LayersSize; } else if (content.id === 'docker-volumes') { content.usage = dockerDf.Volumes.map((v) => v.UsageData.Size).reduce((a,b) => a + b, 0); } else { const [error, usage] = await safe(du(content.path)); if (error) progressCallback({ message: `du error: ${error.message}`}); // can happen if app is installing etc content.usage = usage || 0; } progressCallback({ message: `du of ${JSON.stringify(content)}: ${content.usage}`}); } } if (!safe.fs.writeFileSync(paths.DISK_USAGE_FILE, JSON.stringify({ ts: now, filesystems }, null, 4), 'utf8')) throw new BoxError(BoxError.FS_ERROR, `Could not write du cache file: ${safe.error.message}`); return filesystems; } async function reboot() { await notifications.clearAlert(notifications.ALERT_REBOOT, 'Reboot Required'); const [error] = await safe(shell.promises.sudo([ REBOOT_CMD ], {})); if (error) debug('reboot: could not reboot. %o', error); } async function getInfo() { // https://serverfault.com/questions/92932/how-does-ubuntu-keep-track-of-the-system-restart-required-flag-in-motd const rebootRequired = fs.existsSync('/var/run/reboot-required'); const uptime = safe.fs.readFileSync('/proc/uptime', 'utf8'); const uptimeSecs = parseInt(uptime.split(' ')[0], 10); const sysVendor = safe.fs.readFileSync('/sys/devices/virtual/dmi/id/sys_vendor', 'utf8'); const productName = safe.fs.readFileSync('/sys/devices/virtual/dmi/id/product_name', 'utf8'); const activationTime = (await eventlog.getActivationEvent())?.creationTime || null; return { sysVendor: sysVendor.trim(), productName: productName.trim(), uptimeSecs, rebootRequired, activationTime }; } async function startUpdateDiskUsage() { const taskId = await tasks.add(tasks.TASK_UPDATE_DISK_USAGE, []); tasks.startTask(taskId, {}); return taskId; } async function getLogs(unit, options) { assert.strictEqual(typeof unit, 'string'); assert(options && typeof options === 'object'); if (unit !== 'box') throw new BoxError(BoxError.BAD_FIELD, `No such unit '${unit}'`); const logFile = path.join(paths.LOG_DIR, 'box.log'); const cp = logs.tail([logFile], { lines: options.lines, follow: options.follow }); const logStream = new logs.LogStream({ format: options.format || 'json', source: unit }); logStream.on('close', () => cp.terminate()); // the caller has to call destroy() on logStream. destroy() of Transform emits 'close' cp.stdout.pipe(logStream); return logStream; } // this gets block devices as opposed to mounted filesystems. this is used for configuring backups and volumes in the frontend async function getBlockDevices() { const result = await shell.spawn('lsblk', ['--paths', '--json', '--list', '--fs', '--output', '+rota'], { encoding: 'utf8' }); const info = safe.JSON.parse(result); if (!info) throw new BoxError(BoxError.INTERNAL_ERROR, `failed to parse lsblk: ${safe.error.message}`); const devices = info.blockdevices.filter(d => d.fstype === 'ext4' || d.fstype === 'xfs'); debug(`getBlockDevices: Found ${devices.length} devices. ${devices.map(d => d.name).join(', ')}`); return devices.map(function (d) { return { path: d.name, size: d.fsavail || 0, type: d.fstype, uuid: d.uuid, rota: d.rota, // false (ssd) true (hdd) mountpoint: d.mountpoints ? d.mountpoints.pop() : d.mountpoint // we only support one mountpoint here old lsblk only exposed one via .mountpoint }; }); } async function checkRebootRequired() { const { rebootRequired } = await getInfo(); if (rebootRequired) { await notifications.alert(notifications.ALERT_REBOOT, 'Reboot Required', 'To finish ubuntu security updates, a reboot is necessary.', { persist: true }); } else { await notifications.clearAlert(notifications.ALERT_REBOOT, 'Reboot Required'); } } async function getUbuntuVersion() { const release = safe.fs.readFileSync('/etc/lsb-release', 'utf-8'); if (release === null) throw new BoxError(BoxError.FS_ERROR, safe.error.message); return release.match(/DISTRIB_DESCRIPTION="(.*)"/)[1]; } async function checkUbuntuVersion() { const isXenial = fs.readFileSync('/etc/lsb-release', 'utf-8').includes('16.04'); if (!isXenial) return; await notifications.alert(notifications.ALERT_UPDATE_UBUNTU, 'Ubuntu upgrade required', 'Ubuntu 16.04 has reached end of life and will not receive security and maintenance updates. Please follow https://docs.cloudron.io/guides/upgrade-ubuntu-18/ to upgrade to Ubuntu 18 at the earliest.', { persist: true }); } async function runSystemChecks() { debug('runSystemChecks: checking status'); const checks = [ checkRebootRequired(), checkUbuntuVersion() ]; await Promise.allSettled(checks); } function getProvider() { const provider = safe.fs.readFileSync(paths.PROVIDER_FILE, 'utf8'); return provider ? provider.trim() : 'generic'; } async function getCpus() { return os.cpus(); }