total stats are reported directly as single value

this can happen when the app is installing/uninstalling

CHANGES

@@ -2541,4 +2541,11 @@
 
 [7.3.1]
 * Add cloudlare R2
+* app proxy: fixes to https proxying
+* app links: fix icons
 
+[7.3.2]
+* support: require owner permissions
+* postgresql: fix issue when restoring large dumps
+* graphs: add cpu/disk/network usage
+* graphs: new disk usage UI

package.json

@@ -74,7 +74,7 @@
     "nyc": "^15.1.0"
   },
   "scripts": {
-    "test": "./runTests",
+    "test": "./run-tests",
     "postmerge": "/bin/true",
     "precommit": "/bin/true",
     "prepush": "npm test",

run-tests

@@ -6,7 +6,7 @@ readonly source_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 readonly DATA_DIR="${HOME}/.cloudron_test"
 readonly DEFAULT_TESTS="./src/test/*-test.js ./src/routes/test/*-test.js"
 
-! "${source_dir}/src/test/checkInstall" && exit 1
+! "${source_dir}/src/test/check-install" && exit 1
 
 # cleanup old data dirs some of those docker container data requires sudo to be removed
 echo "=> Provide root password to purge any leftover data in ${DATA_DIR} and load apparmor profile:"
@@ -39,7 +39,7 @@ if [[ -z ${FAST+x} ]]; then
     echo "=> Delete all docker containers first"
     docker ps -qa --filter "label=isCloudronManaged" | xargs --no-run-if-empty docker rm -f
     docker rm -f mysql-server
-    echo "==> To skip this run with: FAST=1 ./runTests"
+    echo "==> To skip this run with: FAST=1 ./run-tests"
 else
     echo "==> WARNING!! Skipping docker container cleanup, the database might not be pristine!"
 fi

setup/start.sh

@@ -59,7 +59,7 @@ mkdir -p "${PLATFORM_DATA_DIR}/mongodb"
 mkdir -p "${PLATFORM_DATA_DIR}/redis"
 mkdir -p "${PLATFORM_DATA_DIR}/addons/mail/banner" \
          "${PLATFORM_DATA_DIR}/addons/mail/dkim"
-mkdir -p "${PLATFORM_DATA_DIR}/collectd/collectd.conf.d"
+mkdir -p "${PLATFORM_DATA_DIR}/collectd"
 mkdir -p "${PLATFORM_DATA_DIR}/logrotate.d"
 mkdir -p "${PLATFORM_DATA_DIR}/acme"
 mkdir -p "${PLATFORM_DATA_DIR}/backup"
@@ -133,7 +133,7 @@ rm -f /etc/sudoers.d/${USER} /etc/sudoers.d/cloudron
 cp "${script_dir}/start/sudoers" /etc/sudoers.d/cloudron
 
 log "Configuring collectd"
-rm -rf /etc/collectd /var/log/collectd.log
+rm -rf /etc/collectd /var/log/collectd.log "${PLATFORM_DATA_DIR}/collectd/collectd.conf.d"
 ln -sfF "${PLATFORM_DATA_DIR}/collectd" /etc/collectd
 cp "${script_dir}/start/collectd/collectd.conf" "${PLATFORM_DATA_DIR}/collectd/collectd.conf"
 systemctl restart collectd

setup/start/collectd/collectd.conf

@@ -187,9 +187,9 @@ LoadPlugin swap
 
        CalculateNum false
        CalculateSum true
-       CalculateAverage true
+       CalculateAverage false
        CalculateMinimum false
-       CalculateMaximum true
+       CalculateMaximum false
        CalculateStddev false
    </Aggregation>
 </Plugin>
@@ -211,23 +211,7 @@ LoadPlugin swap
     Interactive false
 
     Import "df"
-
-    Import "du"
-    <Module du>
-        <Path>
-            Instance maildata
-            Dir "/home/yellowtent/boxdata/mail"
-        </Path>
-        <Path>
-            Instance boxdata
-            Dir "/home/yellowtent/boxdata"
-            Exclude "mail"
-        </Path>
-        <Path>
-            Instance platformdata
-            Dir "/home/yellowtent/platformdata"
-        </Path>
-    </Module>
+    Import "docker-stats"
 </Plugin>
 
 <Plugin write_graphite>
@@ -243,6 +227,3 @@ LoadPlugin swap
    </Node>
 </Plugin>
 
-<Include "/etc/collectd/collectd.conf.d">
-    Filter "*.conf"
-</Include>

setup/start/collectd/docker-stats.py

@@ -0,0 +1,64 @@
+import collectd,os,subprocess,json,re
+
+# https://blog.dbrgn.ch/2017/3/10/write-a-collectd-python-plugin/
+
+def parseSiSize(size):
+    units = {"B": 1, "KB": 10**3, "MB": 10**6, "GB": 10**9, "TB": 10**12}
+    number, unit, _ = re.split('([a-zA-Z]+)', size.upper())
+    return int(float(number)*units[unit])
+
+def parseBinarySize(size):
+    units = {"B": 1, "KIB": 2**10, "MIB": 2**20, "GIB": 2**30, "TIB": 2**40}
+    number, unit, _ = re.split('([a-zA-Z]+)', size.upper())
+    return int(float(number)*units[unit])
+
+def init():
+    collectd.info('custom docker-status plugin initialized')
+
+def read():
+    try:
+        lines = subprocess.check_output('docker stats --format  "{{ json . }}" --no-stream --no-trunc', shell=True).decode('utf-8').strip().split("\n")
+    except Exception as e:
+        collectd.info('\terror getting docker stats: %s' % (str(e)))
+        return 0
+
+    # Sample line
+    # {"BlockIO":"430kB / 676kB","CPUPerc":"0.00%","Container":"7eae5e6f4f11","ID":"7eae5e6f4f11","MemPerc":"59.15%","MemUsage":"45.55MiB / 77MiB","Name":"1062eef3-ec96-4d81-9f02-15b7dd81ccb9","NetIO":"1.5MB / 3.48MB","PIDs":"5"}
+
+    for line in lines:
+        stat = json.loads(line)
+        containerName = stat["Name"] # same as app id
+        networkData = stat["NetIO"].split("/")
+        networkRead = parseSiSize(networkData[0].strip())
+        networkWrite = parseSiSize(networkData[1].strip())
+
+        blockData = stat["BlockIO"].split("/")
+        blockRead = parseSiSize(blockData[0].strip())
+        blockWrite = parseSiSize(blockData[1].strip())
+
+        memUsageData = stat["MemUsage"].split("/")
+        memUsed = parseBinarySize(memUsageData[0].strip())
+        memMax = parseBinarySize(memUsageData[1].strip())
+
+        cpuPercData = stat["CPUPerc"].strip("%")
+        cpuPerc = float(cpuPercData)
+
+        # type comes from https://github.com/collectd/collectd/blob/master/src/types.db and https://collectd.org/wiki/index.php/Data_source
+        val = collectd.Values(type='gauge', plugin='docker-stats', plugin_instance=containerName)
+        val.dispatch(values=[networkRead], type_instance='network-read')
+        val.dispatch(values=[networkWrite], type_instance='network-write')
+        val.dispatch(values=[blockRead], type_instance='blockio-read')
+        val.dispatch(values=[blockWrite], type_instance='blockio-write')
+        val.dispatch(values=[memUsed], type_instance='mem-used')
+        val.dispatch(values=[memMax], type_instance='mem-max')
+        val.dispatch(values=[cpuPerc], type_instance='cpu-perc')
+
+        val = collectd.Values(type='counter', plugin='docker-stats', plugin_instance=containerName)
+        val.dispatch(values=[networkRead], type_instance='network-read')
+        val.dispatch(values=[networkWrite], type_instance='network-write')
+        val.dispatch(values=[blockRead], type_instance='blockio-read')
+        val.dispatch(values=[blockWrite], type_instance='blockio-write')
+
+collectd.register_init(init)
+# see Interval setting in collectd.conf for polling interval
+collectd.register_read(read)

setup/start/collectd/du.py

@@ -1,105 +0,0 @@
-import collectd,os,subprocess,sys,re,time
-
-# https://www.programcreek.com/python/example/106897/collectd.register_read
-
-PATHS = [] # { name, dir, exclude }
-# there is a pattern in carbon/storage-schemas.conf which stores values every 12h for a year
-INTERVAL = 60 * 60 * 12 # twice a day. change values in docker-graphite if you change this
-
-# we used to pass the INTERVAL as a parameter to register_read. however, collectd write_graphite
-# takes a bit to load (tcp connection) and drops the du data. this then means that we have to wait
-# for INTERVAL secs for du data. instead, we just cache the value for INTERVAL instead
-CACHE = dict()
-CACHE_TIME = 0
-
-def du(pathinfo):
-    # -B1 makes du print block sizes and not apparent sizes (to match df which also uses block sizes)
-    dirname = pathinfo['dir']
-    cmd = 'timeout 1800 du -DsB1 "{}"'.format(dirname)
-    if pathinfo['exclude'] != '':
-        cmd += ' --exclude "{}"'.format(pathinfo['exclude'])
-
-    collectd.info('computing size with command: %s' % cmd);
-    try:
-        size = subprocess.check_output(cmd, shell=True).split()[0].decode('utf-8')
-        collectd.info('\tsize of %s is %s (time: %i)' % (dirname, size, int(time.time())))
-        return size
-    except Exception as e:
-        collectd.info('\terror getting the size of %s: %s' % (dirname, str(e)))
-        return 0
-
-def parseSize(size):
-    units = {"B": 1, "KB": 10**3, "MB": 10**6, "GB": 10**9, "TB": 10**12}
-    number, unit, _ = re.split('([a-zA-Z]+)', size.upper())
-    return int(float(number)*units[unit])
-
-def dockerSize():
-    # use --format '{{json .}}' to dump the string. '{{if eq .Type "Images"}}{{.Size}}{{end}}' still creates newlines
-    # https://godoc.org/github.com/docker/go-units#HumanSize is used. so it's 1000 (KB) and not 1024 (KiB)
-    cmd = 'timeout 1800 docker system df --format "{{.Size}}" | head -n1'
-    try:
-        size = subprocess.check_output(cmd, shell=True).strip().decode('utf-8')
-        collectd.info('size of docker images is %s (%s) (time: %i)' % (size, parseSize(size), int(time.time())))
-        return parseSize(size)
-    except Exception as e:
-        collectd.info('error getting docker images size : %s' % str(e))
-        return 0
-
-# configure is called for each module block. this is called before init
-def configure(config):
-    global PATHS
-
-    for child in config.children:
-        if child.key != 'Path':
-            collectd.info('du plugin: Unknown config key "%s"' % key)
-            continue
-
-        pathinfo = { 'name': '', 'dir': '', 'exclude': '' }
-        for node in child.children:
-            if node.key == 'Instance':
-                pathinfo['name'] = node.values[0]
-            elif node.key == 'Dir':
-                pathinfo['dir'] = node.values[0]
-            elif node.key == 'Exclude':
-                pathinfo['exclude'] = node.values[0]
-
-        PATHS.append(pathinfo);
-        collectd.info('du plugin: monitoring %s' % pathinfo['dir']);
-
-def init():
-    global PATHS
-    collectd.info('custom du plugin initialized with %s %s' % (PATHS, sys.version))
-
-def read():
-    global CACHE, CACHE_TIME
-
-    # read from cache if < 12 hours
-    read_cache = (time.time() - CACHE_TIME) < INTERVAL
-
-    if not read_cache:
-        CACHE_TIME = time.time()
-
-    for pathinfo in PATHS:
-        dirname = pathinfo['dir']
-        if read_cache and dirname in CACHE:
-            size = CACHE[dirname]
-        else:
-            size = du(pathinfo)
-            CACHE[dirname] = size
-
-        # type comes from https://github.com/collectd/collectd/blob/master/src/types.db
-        val = collectd.Values(type='capacity', plugin='du', plugin_instance=pathinfo['name'])
-        val.dispatch(values=[size], type_instance='usage')
-
-    if read_cache and 'docker' in CACHE:
-        size = CACHE['docker']
-    else:
-        size = dockerSize()
-        CACHE['docker'] = size
-
-    val = collectd.Values(type='capacity', plugin='du', plugin_instance='docker')
-    val.dispatch(values=[size], type_instance='usage')
-
-collectd.register_init(init)
-collectd.register_config(configure)
-collectd.register_read(read)

setup/start/sudoers

@@ -19,9 +19,6 @@ yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/rmaddondir.sh
 Defaults!/home/yellowtent/box/src/scripts/reboot.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/reboot.sh
 
-Defaults!/home/yellowtent/box/src/scripts/configurecollectd.sh env_keep="HOME BOX_ENV"
-yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/configurecollectd.sh
-
 Defaults!/home/yellowtent/box/src/scripts/collectlogs.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/collectlogs.sh
 
@@ -68,5 +65,7 @@ yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/rmmount.sh
 Defaults!/home/yellowtent/box/src/scripts/remountmount.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/remountmount.sh
 
-cloudron-support ALL=(ALL) NOPASSWD: ALL
+Defaults!/home/yellowtent/box/src/scripts/du.sh env_keep="HOME BOX_ENV"
+yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/du.sh
 
+cloudron-support ALL=(ALL) NOPASSWD: ALL

src/apphealthmonitor.js

@@ -73,7 +73,7 @@ async function checkAppHealth(app, options) {
     let healthCheckUrl, host;
     if (app.manifest.id === constants.PROXY_APP_APPSTORE_ID) {
         healthCheckUrl = app.upstreamUri;
-        host = '';
+        host = new URL(app.upstreamUri).host; // includes port
     } else {
         const [error, data] = await safe(docker.inspect(app.containerId));
         if (error || !data || !data.State) return await setHealth(app, apps.HEALTH_ERROR);
@@ -88,6 +88,7 @@ async function checkAppHealth(app, options) {
 
     const [healthCheckError, response] = await safe(superagent
         .get(healthCheckUrl)
+        .disableTLSCerts() // for app proxy
         .set('Host', host) // required for some apache configs with rewrite rules
         .set('User-Agent', 'Mozilla (CloudronHealth)') // required for some apps (e.g. minio)
         .redirects(0)

src/applinks.js

@@ -12,14 +12,14 @@ exports = module.exports = {
 
 const assert = require('assert'),
     apps = require('./apps.js'),
-    database = require('./database.js'),
     BoxError = require('./boxerror.js'),
-    uuid = require('uuid'),
+    database = require('./database.js'),
+    debug = require('debug')('box:applinks'),
+    jsdom = require('jsdom'),
     safe = require('safetydance'),
     superagent = require('superagent'),
-    validator = require('validator'),
-    jsdom = require('jsdom'),
-    debug = require('debug')('box:applinks');
+    uuid = require('uuid'),
+    validator = require('validator');
 
 const APPLINKS_FIELDS= [ 'id', 'accessRestrictionJson', 'creationTime', 'updateTime', 'ts', 'label', 'tagsJson', 'icon', 'upstreamUri' ].join(',');
 
@@ -38,6 +38,21 @@ function postProcess(result) {
     result.ts = new Date(result.ts).getTime();
 
     result.icon = result.icon ? result.icon : null;
+
+}
+
+function validateUpstreamUri(upstreamUri) {
+    assert.strictEqual(typeof upstreamUri, 'string');
+
+    if (!upstreamUri) return new BoxError(BoxError.BAD_FIELD, 'upstreamUri cannot be empty');
+
+    if (!upstreamUri.includes('://')) return new BoxError(BoxError.BAD_FIELD, 'upstreamUri has no schema');
+
+    const uri = safe(() => new URL(upstreamUri));
+    if (!uri) return new BoxError(BoxError.BAD_FIELD, `upstreamUri is invalid: ${safe.error.message}`);
+    if (uri.protocol !== 'http:' && uri.protocol !== 'https:') return new BoxError(BoxError.BAD_FIELD, 'upstreamUri has an unsupported scheme');
+
+    return null;
 }
 
 async function list() {
@@ -58,28 +73,40 @@ async function listByUser(user) {
 async function detectMetaInfo(applink) {
     assert.strictEqual(typeof applink, 'object');
 
-    const [error, response] = await safe(superagent.get(applink.upstreamUri));
+    const [error, response] = await safe(superagent.get(applink.upstreamUri).set('User-Agent', 'Mozilla'));
     if (error || !response.text) throw new BoxError(BoxError.BAD_FIELD, 'cannot fetch upstream uri for favicon and label');
 
-    // fixup upstreamUri to match the redirect
-    if (response.redirects && response.redirects.length) {
-        debug(`detectMetaInfo: found redirect from ${applink.upstreamUri} to ${response.redirects[0]}`);
-        applink.upstreamUri = response.redirects[0];
-    }
-
     if (applink.favicon && applink.label) return;
 
+    // set redirected URI if any for favicon url
+    const redirectUri = (response.redirects && response.redirects.length) ? response.redirects[0] : null;
+
     const dom = new jsdom.JSDOM(response.text);
     if (!applink.icon) {
         let favicon = '';
         if (dom.window.document.querySelector('link[rel="apple-touch-icon"]')) favicon = dom.window.document.querySelector('link[rel="apple-touch-icon"]').href ;
         if (!favicon.endsWith('.png') && dom.window.document.querySelector('meta[name="msapplication-TileImage"]')) favicon = dom.window.document.querySelector('meta[name="msapplication-TileImage"]').content ;
         if (!favicon.endsWith('.png') && dom.window.document.querySelector('link[rel="shortcut icon"]')) favicon = dom.window.document.querySelector('link[rel="shortcut icon"]').href ;
-        if (!favicon.endsWith('.png') && dom.window.document.querySelector('link[rel="icon"]')) favicon = dom.window.document.querySelector('link[rel="icon"]').href ;
+        if (!favicon.endsWith('.png') && dom.window.document.querySelector('link[rel="icon"]')) {
+            let iconElements = dom.window.document.querySelectorAll('link[rel="icon"]');
+            if (iconElements.length) {
+                favicon = iconElements[0].href; // choose first one for a start
+
+                // check if we have sizes attributes and then choose the largest one
+                iconElements = Array.from(iconElements).filter(function (e) {
+                    return e.attributes.getNamedItem('sizes') && e.attributes.getNamedItem('sizes').value;
+                }).sort(function (a, b) {
+                    return parseInt(b.attributes.getNamedItem('sizes').value.split('x')[0]) - parseInt(a.attributes.getNamedItem('sizes').value.split('x')[0]);
+                });
+                if (iconElements.length) favicon = iconElements[0].href;
+            }
+        }
         if (!favicon.endsWith('.png') && dom.window.document.querySelector('meta[itemprop="image"]')) favicon = dom.window.document.querySelector('meta[itemprop="image"]').content;
 
         if (favicon) {
-            if (favicon.startsWith('/')) favicon = applink.upstreamUri + favicon;
+            if (favicon.startsWith('/')) favicon = (redirectUri || applink.upstreamUri) + favicon;
+
+            debug(`detectMetaInfo: found icon: ${favicon}`);
 
             const [error, response] = await safe(superagent.get(favicon));
             if (error) console.error(`Failed to fetch icon ${favicon}: `, error);
@@ -103,6 +130,9 @@ async function add(applink) {
 
     debug(`add: ${applink.upstreamUri}`, applink);
 
+    let error = validateUpstreamUri(applink.upstreamUri);
+    if (error) throw error;
+
     if (applink.icon) {
         if (!validator.isBase64(applink.icon)) throw new BoxError(BoxError.BAD_FIELD, 'icon is not base64');
         applink.icon = Buffer.from(applink.icon, 'base64');
@@ -122,7 +152,7 @@ async function add(applink) {
     const query = 'INSERT INTO applinks (id, accessRestrictionJson, label, tagsJson, icon, upstreamUri) VALUES (?, ?, ?, ?, ?, ?)';
     const args = [ data.id, data.accessRestrictionJson, data.label, data.tagsJson, data.icon, data.upstreamUri ];
 
-    const [error] = await safe(database.query(query, args));
+    [error] = await safe(database.query(query, args));
     if (error) throw error;
 
     return data.id;
@@ -132,7 +162,7 @@ async function get(applinkId) {
     assert.strictEqual(typeof applinkId, 'string');
 
     const result = await database.query(`SELECT ${APPLINKS_FIELDS} FROM applinks WHERE id = ?`, [ applinkId ]);
-    if (result.length === 0) throw new BoxError(BoxError.NOT_FOUND, 'Applink not found');
+    if (result.length === 0) return null;
 
     postProcess(result[0]);
 
@@ -144,7 +174,10 @@ async function update(applinkId, applink) {
     assert.strictEqual(typeof applink, 'object');
     assert.strictEqual(typeof applink.upstreamUri, 'string');
 
-    debug(`update: ${applinkId} ${applink.upstreamUri}`, applink);
+    debug(`update: ${applink.upstreamUri}`, applink);
+
+    let error = validateUpstreamUri(applink.upstreamUri);
+    if (error) throw error;
 
     if (applink.icon) {
         if (!validator.isBase64(applink.icon)) throw new BoxError(BoxError.BAD_FIELD, 'icon is not base64');
@@ -163,9 +196,7 @@ async function update(applinkId, applink) {
 async function remove(applinkId) {
     assert.strictEqual(typeof applinkId, 'string');
 
-    debug(`remove: ${applinkId}`);
-
-    const result = await database.query(`DELETE FROM applinks WHERE id = ?`, [ applinkId ]);
+    const result = await database.query('DELETE FROM applinks WHERE id = ?', [ applinkId ]);
     if (result.affectedRows !== 1) throw new BoxError(BoxError.NOT_FOUND, 'Applink not found');
 }
 
@@ -173,6 +204,7 @@ async function getIcon(applinkId) {
     assert.strictEqual(typeof applinkId, 'string');
 
     const applink = await get(applinkId);
+    if (!applink) throw new BoxError(BoxError.NOT_FOUND, 'Applink not found');
 
     return applink.icon;
 }

src/apps.js

@@ -218,7 +218,6 @@ function validatePortBindings(portBindings, manifest) {
         993, /* imaps */
         995, /* pop3s */
         2003, /* graphite (lo) */
-        2004, /* graphite (lo) */
         2514, /* cloudron-syslog (lo) */
         constants.PORT, /* app server (lo) */
         constants.AUTHWALL_PORT, /* protected sites */
@@ -229,7 +228,6 @@ function validatePortBindings(portBindings, manifest) {
         4190, /* managesieve */
         5349, /* turn,stun TLS */
         8000, /* ESXi monitoring */
-        8417, /* graphite (lo) */
     ];
 
     const RESERVED_PORT_RANGES = [
@@ -771,7 +769,7 @@ function isAdmin(user) {
 }
 
 function isOperator(app, user) {
-    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof app, 'object');  // IMPORTANT: can also be applink
     assert.strictEqual(typeof user, 'object');
 
     if (!app.operators) return isAdmin(user);
@@ -784,7 +782,7 @@ function isOperator(app, user) {
 }
 
 function canAccess(app, user) {
-    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof app, 'object'); // IMPORTANT: can also be applink
     assert.strictEqual(typeof user, 'object');
 
     if (app.accessRestriction === null) return true;
@@ -2174,20 +2172,18 @@ async function importApp(app, data, auditSource) {
 
     const appId = app.id;
 
-    // all fields are optional
-    data.remotePath = data.remotePath || null;
-    data.backupFormat = data.backupFormat || null;
-    data.backupConfig = data.backupConfig || null;
     const { remotePath, backupFormat, backupConfig } = data;
 
-    let error = backupFormat ? validateBackupFormat(backupFormat) : null;
+    let error = checkAppState(app, exports.ISTATE_PENDING_IMPORT);
     if (error) throw error;
 
-    error = checkAppState(app, exports.ISTATE_PENDING_IMPORT);
-    if (error) throw error;
+    let restoreConfig;
 
-    // TODO: make this smarter to do a read-only test and check if the file exists in the storage backend
-    if (backupConfig) {
+    if (data.remotePath) { // if not provided, we import in-place
+        error = validateBackupFormat(backupFormat);
+        if (error) throw error;
+
+        // TODO: make this smarter to do a read-only test and check if the file exists in the storage backend
         if (mounts.isManagedProvider(backupConfig.provider)) {
             error = mounts.validateMountOptions(backupConfig.provider, backupConfig.mountOptions);
             if (error) throw error;
@@ -2203,18 +2199,18 @@ async function importApp(app, data, auditSource) {
         }
         error = await backups.testProviderConfig(backupConfig);
         if (error) throw error;
-    }
 
-    if (backupConfig) {
         if ('password' in backupConfig) {
             backupConfig.encryption = backups.generateEncryptionKeysSync(backupConfig.password);
             delete backupConfig.password;
         } else {
             backupConfig.encryption = null;
         }
-    }
 
-    const restoreConfig = { remotePath, backupFormat, backupConfig };
+        restoreConfig = { remotePath, backupFormat, backupConfig };
+    } else {
+        restoreConfig = { remotePath: null };
+    }
 
     const task = {
         args: {

src/apptask.js

@@ -17,7 +17,6 @@ const apps = require('./apps.js'),
     AuditSource = require('./auditsource.js'),
     backuptask = require('./backuptask.js'),
     BoxError = require('./boxerror.js'),
-    collectd = require('./collectd.js'),
     constants = require('./constants.js'),
     debug = require('debug')('box:apptask'),
     df = require('@sindresorhus/df'),
@@ -45,10 +44,6 @@ const MV_VOLUME_CMD = path.join(__dirname, 'scripts/mvvolume.sh'),
     LOGROTATE_CONFIG_EJS = fs.readFileSync(__dirname + '/logrotate.ejs', { encoding: 'utf8' }),
     CONFIGURE_LOGROTATE_CMD = path.join(__dirname, 'scripts/configurelogrotate.sh');
 
-// https://rootlesscontaine.rs/getting-started/common/cgroup2/#checking-whether-cgroup-v2-is-already-enabled
-const CGROUP_VERSION = fs.existsSync('/sys/fs/cgroup/cgroup.controllers') ? '2' : '1';
-const COLLECTD_CONFIG_EJS = fs.readFileSync(`${__dirname}/collectd/app_cgroup_v${CGROUP_VERSION}.ejs`, { encoding: 'utf8' });
-
 function makeTaskError(error, app) {
     assert(error instanceof BoxError);
     assert.strictEqual(typeof app, 'object');
@@ -98,7 +93,6 @@ async function createContainer(app) {
 
     // re-generate configs that rely on container id
     await addLogrotateConfig(app);
-    await addCollectdProfile(app);
 }
 
 async function deleteContainers(app, options) {
@@ -108,7 +102,6 @@ async function deleteContainers(app, options) {
     debug('deleteContainer: deleting app containers (app, scheduler)');
 
     // remove configs that rely on container id
-    await removeCollectdProfile(app);
     await removeLogrotateConfig(app);
     await docker.stopContainers(app.id);
     await docker.deleteContainers(app.id, options);
@@ -161,20 +154,6 @@ async function deleteAppDir(app, options) {
     }
 }
 
-async function addCollectdProfile(app) {
-    assert.strictEqual(typeof app, 'object');
-
-    const appDataDir = await apps.getStorageDir(app);
-    const collectdConf = ejs.render(COLLECTD_CONFIG_EJS, { appId: app.id, containerId: app.containerId, appDataDir });
-    await collectd.addProfile(app.id, collectdConf);
-}
-
-async function removeCollectdProfile(app) {
-    assert.strictEqual(typeof app, 'object');
-
-    await collectd.removeProfile(app.id);
-}
-
 async function addLogrotateConfig(app) {
     assert.strictEqual(typeof app, 'object');
 
@@ -690,9 +669,6 @@ async function start(app, args, progressCallback) {
         await docker.startContainer(app.id);
     }
 
-    await progressCallback({ percent: 60, message: 'Adding collectd profile' });
-    await addCollectdProfile(app);
-
     // stopped apps do not renew certs. currently, we don't do DNS to not overwrite existing user settings
     await progressCallback({ percent: 80, message: 'Configuring reverse proxy' });
     await reverseProxy.configureApp(app, AuditSource.APPTASK);
@@ -713,9 +689,6 @@ async function stop(app, args, progressCallback) {
     await progressCallback({ percent: 50, message: 'Stopping app services' });
     await services.stopAppServices(app);
 
-    await progressCallback({ percent: 80, message: 'Removing collectd profile' });
-    await removeCollectdProfile(app);
-
     await progressCallback({ percent: 100, message: 'Done' });
     await updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null });
 }
@@ -733,9 +706,6 @@ async function restart(app, args, progressCallback) {
         await docker.restartContainer(app.id);
     }
 
-    await progressCallback({ percent: 60, message: 'Adding collectd profile' });
-    await addCollectdProfile(app);
-
     // stopped apps do not renew certs. currently, we don't do DNS to not overwrite existing user settings
     await progressCallback({ percent: 80, message: 'Configuring reverse proxy' });
     await reverseProxy.configureApp(app, AuditSource.APPTASK);

src/backupcleaner.js

@@ -15,7 +15,6 @@ const apps = require('./apps.js'),
     constants = require('./constants.js'),
     debug = require('debug')('box:backupcleaner'),
     moment = require('moment'),
-    mounts = require('./mounts.js'),
     path = require('path'),
     paths = require('./paths.js'),
     safe = require('safetydance'),
@@ -272,12 +271,9 @@ async function run(progressCallback) {
 
     const backupConfig = await settings.getBackupConfig();
 
-    if (mounts.isManagedProvider(backupConfig.provider) || backupConfig.provider === 'mountpoint') {
-        const hostPath = mounts.isManagedProvider(backupConfig.provider) ? paths.MANAGED_BACKUP_MOUNT_DIR : backupConfig.mountPoint;
-        const status = await mounts.getStatus(backupConfig.provider, hostPath); // { state, message }
-        debug(`clean: mount point status is ${JSON.stringify(status)}`);
-        if (status.state !== 'active') throw new BoxError(BoxError.MOUNT_ERROR, `Backup endpoint is not mounted: ${status.message}`);
-    }
+    const status = await storage.api(backupConfig.provider).getBackupProviderStatus(backupConfig);
+    debug(`clean: mount point status is ${JSON.stringify(status)}`);
+    if (status.state !== 'active') throw new BoxError(BoxError.MOUNT_ERROR, `Backup endpoint is not mounted: ${status.message}`);
 
     if (backupConfig.retentionPolicy.keepWithinSecs < 0) {
         debug('cleanup: keeping all backups');

src/backupformat/rsync.js

@@ -27,7 +27,7 @@ function getBackupFilePath(backupConfig, remotePath) {
     assert.strictEqual(typeof backupConfig, 'object');
     assert.strictEqual(typeof remotePath, 'string');
 
-    const rootPath = storage.api(backupConfig.provider).getRootPath(backupConfig);
+    const rootPath = storage.api(backupConfig.provider).getBackupRootPath(backupConfig);
     return path.join(rootPath, remotePath);
 }
 

src/backupformat/tgz.js

@@ -23,7 +23,7 @@ function getBackupFilePath(backupConfig, remotePath) {
     assert.strictEqual(typeof backupConfig, 'object');
     assert.strictEqual(typeof remotePath, 'string');
 
-    const rootPath = storage.api(backupConfig.provider).getRootPath(backupConfig);
+    const rootPath = storage.api(backupConfig.provider).getBackupRootPath(backupConfig);
 
     const fileType = backupConfig.encryption ? '.tar.gz.enc' : '.tar.gz';
     return path.join(rootPath, remotePath + fileType);

src/backups.js

@@ -18,8 +18,6 @@ exports = module.exports = {
     injectPrivateFields,
     removePrivateFields,
 
-    configureCollectd,
-
     generateEncryptionKeysSync,
 
     getSnapshotInfo,
@@ -44,15 +42,12 @@ exports = module.exports = {
 
 const assert = require('assert'),
     BoxError = require('./boxerror.js'),
-    collectd = require('./collectd.js'),
     constants = require('./constants.js'),
     CronJob = require('cron').CronJob,
     crypto = require('crypto'),
     database = require('./database.js'),
     debug = require('debug')('box:backups'),
-    ejs = require('ejs'),
     eventlog = require('./eventlog.js'),
-    fs = require('fs'),
     hat = require('./hat.js'),
     locker = require('./locker.js'),
     path = require('path'),
@@ -62,8 +57,6 @@ const assert = require('assert'),
     storage = require('./storage.js'),
     tasks = require('./tasks.js');
 
-const COLLECTD_CONFIG_EJS = fs.readFileSync(__dirname + '/collectd/cloudron-backup.ejs', { encoding: 'utf8' });
-
 const BACKUPS_FIELDS = [ 'id', 'remotePath', 'label', 'identifier', 'creationTime', 'packageVersion', 'type', 'dependsOnJson', 'state', 'manifestJson', 'format', 'preserveSecs', 'encryptionVersion' ];
 
 function postProcess(result) {
@@ -320,17 +313,6 @@ async function startCleanupTask(auditSource) {
     return taskId;
 }
 
-async function configureCollectd(backupConfig) {
-    assert.strictEqual(typeof backupConfig, 'object');
-
-    if (backupConfig.provider === 'filesystem') {
-        const collectdConf = ejs.render(COLLECTD_CONFIG_EJS, { backupDir: backupConfig.backupFolder });
-        await collectd.addProfile('cloudron-backup', collectdConf);
-    } else {
-        await collectd.removeProfile('cloudron-backup');
-    }
-}
-
 async function testConfig(backupConfig) {
     assert.strictEqual(typeof backupConfig, 'object');
 

src/backuptask.js

@@ -47,6 +47,41 @@ function canBackupApp(app) {
             app.installationState === apps.ISTATE_PENDING_UPDATE; // called from apptask
 }
 
+// binary units (non SI) 1024 based
+function prettyBytes(bytes) {
+    assert.strictEqual(typeof bytes, 'number');
+
+    const i = Math.floor(Math.log(bytes) / Math.log(1024)),
+        sizes = ['B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'];
+
+    return (bytes / Math.pow(1024, i)).toFixed(2) * 1 + '' + sizes[i];
+}
+
+async function checkPreconditions(backupConfig, dataLayout) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
+
+    // check mount status before uploading
+    const status = await storage.api(backupConfig.provider).getBackupProviderStatus(backupConfig);
+    debug(`upload: mount point status is ${JSON.stringify(status)}`);
+    if (status.state !== 'active') throw new BoxError(BoxError.MOUNT_ERROR, `Backup endpoint is not mounted: ${status.message}`);
+
+    // check availabe size. this requires root for df to work
+    const df = await storage.api(backupConfig.provider).getAvailableSize(backupConfig);
+    let used = 0;
+    for (const localPath of dataLayout.localPaths()) {
+        debug(`checkPreconditions: getting disk usage of ${localPath}`);
+        const result = safe.child_process.execSync(`du -Dsb ${localPath}`, { encoding: 'utf8' });
+        if (!result) throw new BoxError(BoxError.FS_ERROR, `du error: ${safe.error.message}`);
+        used += parseInt(result, 10);
+    }
+
+    debug(`checkPreconditions: ${used} bytes`);
+
+    const needed = 0.6 * used + (1024 * 1024 * 1024); // check if there is atleast 1GB left afterwards. aim for 60% because rsync/tgz won't need full 100%
+    if (df.available <= needed) throw new BoxError(BoxError.FS_ERROR, `Not enough disk space for backup. Needed: ${prettyBytes(needed)} Available: ${prettyBytes(df.available)}`);
+}
+
 // this function is called via backupupload (since it needs root to traverse app's directory)
 async function upload(remotePath, format, dataLayoutString, progressCallback) {
     assert.strictEqual(typeof remotePath, 'string');
@@ -58,7 +93,8 @@ async function upload(remotePath, format, dataLayoutString, progressCallback) {
 
     const dataLayout = DataLayout.fromString(dataLayoutString);
     const backupConfig = await settings.getBackupConfig();
-    await storage.api(backupConfig.provider).checkPreconditions(backupConfig, dataLayout);
+
+    await checkPreconditions(backupConfig, dataLayout);
 
     await backupFormat.api(format).upload(backupConfig, remotePath, dataLayout, progressCallback);
 }

src/cloudron.js

@@ -19,6 +19,8 @@ exports = module.exports = {
     renewCerts,
     syncDnsRecords,
 
+    updateDiskUsage,
+
     runSystemChecks
 };
 
@@ -26,7 +28,6 @@ const apps = require('./apps.js'),
     appstore = require('./appstore.js'),
     assert = require('assert'),
     AuditSource = require('./auditsource.js'),
-    backups = require('./backups.js'),
     BoxError = require('./boxerror.js'),
     branding = require('./branding.js'),
     constants = require('./constants.js'),
@@ -108,12 +109,6 @@ async function runStartupTasks() {
     // stop all the systemd tasks
     tasks.push(platform.stopAllTasks);
 
-    // this configures collectd to collect backup storage metrics if filesystem is used. This is also triggerd when the settings change with the rest api
-    tasks.push(async function () {
-        const backupConfig = await settings.getBackupConfig();
-        await backups.configureCollectd(backupConfig);
-    });
-
     // always generate webadmin config since we have no versioning mechanism for the ejs
     tasks.push(async function () {
         if (!settings.dashboardDomain()) return;
@@ -351,3 +346,9 @@ async function syncDnsRecords(options) {
     tasks.startTask(taskId, {});
     return taskId;
 }
+
+async function updateDiskUsage() {
+    const taskId = await tasks.add(tasks.TASK_UPDATE_DISK_USAGE, []);
+    tasks.startTask(taskId, {});
+    return taskId;
+}

src/collectd.js

@@ -1,43 +0,0 @@
-'use strict';
-
-exports = module.exports = {
-    addProfile,
-    removeProfile
-};
-
-const assert = require('assert'),
-    BoxError = require('./boxerror.js'),
-    debug = require('debug')('collectd'),
-    path = require('path'),
-    paths = require('./paths.js'),
-    safe = require('safetydance'),
-    shell = require('./shell.js');
-
-const CONFIGURE_COLLECTD_CMD = path.join(__dirname, 'scripts/configurecollectd.sh');
-
-async function addProfile(name, profile) {
-    assert.strictEqual(typeof name, 'string');
-    assert.strictEqual(typeof profile, 'string');
-
-    const configFilePath = path.join(paths.COLLECTD_APPCONFIG_DIR, `${name}.conf`);
-
-    // skip restarting collectd if the profile already exists with the same contents
-    const currentProfile = safe.fs.readFileSync(configFilePath, 'utf8') || '';
-    if (currentProfile === profile) return;
-
-    if (!safe.fs.writeFileSync(configFilePath, profile)) throw new BoxError(BoxError.FS_ERROR, `Error writing collectd config: ${safe.error.message}`);
-
-    const [error] = await safe(shell.promises.sudo('addCollectdProfile', [ CONFIGURE_COLLECTD_CMD, 'add', name ], {}));
-    if (error) throw new BoxError(BoxError.COLLECTD_ERROR, 'Could not add collectd config');
-}
-
-async function removeProfile(name) {
-    assert.strictEqual(typeof name, 'string');
-
-    if (!safe.fs.unlinkSync(path.join(paths.COLLECTD_APPCONFIG_DIR, `${name}.conf`))) {
-        if (safe.error.code !== 'ENOENT') debug('Error removing collectd profile', safe.error);
-    }
-
-    const [error] = await safe(shell.promises.sudo('removeCollectdProfile', [ CONFIGURE_COLLECTD_CMD, 'remove', name ], {}));
-    if (error) throw new BoxError(BoxError.COLLECTD_ERROR, 'Could not remove collectd config');
-}

src/collectd/app_cgroup_v1.ejs

@@ -1,22 +0,0 @@
-LoadPlugin "table"
-<Plugin table>
-  <Table "/sys/fs/cgroup/memory/docker/<%= containerId %>/memory.memsw.usage_in_bytes">
-    Instance "<%= appId %>-memory"
-    Separator "\\n"
-    <Result>
-      Type gauge
-      InstancePrefix "memsw_usage_in_bytes"
-      ValuesFrom 0
-    </Result>
-  </Table>
-</Plugin>
-
-<Plugin python>
-  <Module du>
-    <Path>
-        Instance "<%= appId %>"
-        Dir "<%= appDataDir %>"
-    </Path>
-  </Module>
-</Plugin>
-

src/collectd/app_cgroup_v2.ejs

@@ -1,32 +0,0 @@
-LoadPlugin "table"
-<Plugin table>
-  <Table "/sys/fs/cgroup/docker/<%= containerId %>/memory.current">
-    Instance "<%= appId %>-memory"
-    Separator " \\n"
-    <Result>
-      Type gauge
-      InstancePrefix "memory_current"
-      ValuesFrom 0
-    </Result>
-  </Table>
-
-  <Table "/sys/fs/cgroup/docker/<%= containerId %>/memory.swap.current">
-    Instance "<%= appId %>-memory"
-    Separator "\\n"
-    <Result>
-      Type gauge
-      InstancePrefix "memory_swap_current"
-      ValuesFrom 0
-    </Result>
-  </Table>
-</Plugin>
-
-<Plugin python>
-  <Module du>
-    <Path>
-        Instance "<%= appId %>"
-        Dir "<%= appDataDir %>"
-    </Path>
-  </Module>
-</Plugin>
-

src/cron.js

@@ -122,7 +122,7 @@ async function startJobs() {
 
     gJobs.cleanupEventlog = new CronJob({
         cronTime: '00 */30 * * * *', // every 30 minutes
-        onTick: async () => await safe(eventlog.cleanup({ creationTime: new Date(Date.now() - 60 * 60 * 24 * 10 * 1000) }), { debug }), // 10 days ago
+        onTick: async () => await safe(eventlog.cleanup({ creationTime: new Date(Date.now() - 60 * 60 * 24 * 60 * 1000) }), { debug }), // 60 days ago
         start: true
     });
 

src/directoryserver.js

@@ -143,6 +143,26 @@ async function authorize(req, res, next) {
     return next();
 }
 
+// https://ldapwiki.com/wiki/RootDSE / RFC 4512 - ldapsearch -x -h "${CLOUDRON_LDAP_SERVER}" -p "${CLOUDRON_LDAP_PORT}" -b  "" -s base
+// ldapjs seems to call this handler for everything when search === ''
+async function maybeRootDSE(req, res, next) {
+    debug(`maybeRootDSE: requested with scope:${req.scope} dn:${req.dn.toString()}`);
+
+    if (req.scope !== 'base') return next(new ldap.NoSuchObjectError()); // per the spec, rootDSE search require base scope
+    if (!req.dn || req.dn.toString() !== '') return next(new ldap.NoSuchObjectError());
+
+    res.send({
+        dn: '',
+        attributes: {
+            objectclass: [ 'RootDSE', 'top', 'OpenLDAProotDSE' ],
+            supportedLDAPVersion: '3',
+            vendorName: 'Cloudron LDAP',
+            vendorVersion: '1.0.0'
+        }
+    });
+    res.end();
+}
+
 async function userSearch(req, res, next) {
     debug('user search: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id);
 
@@ -329,6 +349,8 @@ async function start() {
         res.end();
     });
 
+    gServer.search('', maybeRootDSE); // when '', it seems the callback is called for everything else
+
     // just log that an attempt was made to unknown route, this helps a lot during app packaging
     gServer.use(function(req, res, next) {
         debug('not handled: dn %s, scope %s, filter %s (from %s)', req.dn ? req.dn.toString() : '-', req.scope, req.filter ? req.filter.toString() : '-', req.connection.ldap.id);

src/dns/digitalocean.js

@@ -205,6 +205,11 @@ async function wait(domainObject, subdomain, type, value, options) {
     await waitForDns(fqdn, domainObject.zoneName, type, value, options);
 }
 
+// https://stackoverflow.com/questions/14313183/javascript-regex-how-do-i-check-if-the-string-is-ascii-only
+function isASCII(str) {
+    return /^[\x00-\x7F]*$/.test(str);
+}
+
 async function verifyDomainConfig(domainObject) {
     assert.strictEqual(typeof domainObject, 'object');
 
@@ -212,6 +217,7 @@ async function verifyDomainConfig(domainObject) {
         zoneName = domainObject.zoneName;
 
     if (!domainConfig.token || typeof domainConfig.token !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'token must be a non-empty string');
+    if (!isASCII(domainConfig.token)) throw new BoxError(BoxError.BAD_FIELD, 'token contains invalid characters');
 
     const ip = '127.0.0.1';
 

src/docker.js

@@ -8,6 +8,7 @@ exports = module.exports = {
     ping,
 
     info,
+    df,
     downloadImage,
     createContainer,
     startContainer,
@@ -630,6 +631,12 @@ async function info() {
     return result;
 }
 
+async function df() {
+    const [error, result] = await safe(gConnection.df());
+    if (error) throw new BoxError(BoxError.DOCKER_ERROR, 'Error connecting to docker');
+    return result;
+}
+
 async function update(name, memory, memorySwap) {
     assert.strictEqual(typeof name, 'string');
     assert.strictEqual(typeof memory, 'number');

src/graphs.js

@@ -2,79 +2,85 @@
 
 exports = module.exports = {
     getSystem,
-    getByApp
+    getContainerStats
 };
 
 const apps = require('./apps.js'),
     assert = require('assert'),
     BoxError = require('./boxerror.js'),
-    fs = require('fs'),
+    docker = require('./docker.js'),
+    os = require('os'),
     safe = require('safetydance'),
-    superagent = require('superagent'),
-    system = require('./system.js');
+    services = require('./services.js'),
+    superagent = require('superagent');
 
-// for testing locally: curl 'http://127.0.0.1:8417/graphite-web/render?format=json&from=-1min&target=absolute(collectd.localhost.du-docker.capacity-usage)'
-// the datapoint is (value, timestamp) https://buildmedia.readthedocs.org/media/pdf/graphite/0.9.16/graphite.pdf
-const GRAPHITE_RENDER_URL = 'http://127.0.0.1:8417/graphite-web/render';
+// for testing locally: curl 'http://${graphite-ip}:8000/graphite-web/render?format=json&from=-1min&target=absolute(collectd.localhost.du-docker.capacity-usage)'
+// the datapoint is (value, timestamp) https://graphite.readthedocs.io/en/latest/
+async function getGraphiteUrl() {
+    const [error, result] = await safe(docker.inspect('graphite'));
+    if (error && error.reason === BoxError.NOT_FOUND) return { status: exports.SERVICE_STATUS_STOPPED };
+    if (error) throw error;
 
-// https://rootlesscontaine.rs/getting-started/common/cgroup2/#checking-whether-cgroup-v2-is-already-enabled
-const CGROUP_VERSION = fs.existsSync('/sys/fs/cgroup/cgroup.controllers') ? '2' : '1';
+    const ip = safe.query(result, 'NetworkSettings.Networks.cloudron.IPAddress', null);
+    if (!ip) throw new BoxError(BoxError.INACTIVE, 'Error getting IP of graphite service');
 
-async function getByApp(app, fromMinutes, noNullPoints) {
-    assert.strictEqual(typeof app, 'object');
+    return `http://${ip}:8000/graphite-web/render`;
+}
+
+async function getContainerStats(name, fromMinutes, noNullPoints) {
+    assert.strictEqual(typeof name, 'string');
     assert.strictEqual(typeof fromMinutes, 'number');
     assert.strictEqual(typeof noNullPoints, 'boolean');
 
     const timeBucketSize = fromMinutes > (24 * 60) ? (6*60) : 5;
+    const graphiteUrl = await getGraphiteUrl();
 
-    const memoryQuery = {
-        target: null, // filled below
-        format: 'json',
-        from: `-${fromMinutes}min`,
-        until: 'now'
-    };
-    if (CGROUP_VERSION === '1') {
-        memoryQuery.target = `summarize(collectd.localhost.table-${app.id}-memory.gauge-memsw_usage_in_bytes, "${timeBucketSize}min", "avg")`;
-    } else {
-        memoryQuery.target = `summarize(sum(collectd.localhost.table-${app.id}-memory.gauge-memory_current, collectd.localhost.table-${app.id}-memory.gauge-memory_swap_current), "${timeBucketSize}min", "avg")`;
-    }
-    if (noNullPoints) memoryQuery.noNullPoints = true;
+    const targets = [
+        `summarize(collectd.localhost.docker-stats-${name}.gauge-cpu-perc, "${timeBucketSize}min", "avg")`,
+        `summarize(collectd.localhost.docker-stats-${name}.gauge-mem-used, "${timeBucketSize}min", "avg")`,
+        // `summarize(collectd.localhost.docker-stats-${name}.gauge-mem-max, "${timeBucketSize}min", "avg")`,
+        `summarize(collectd.localhost.docker-stats-${name}.counter-blockio-read, "${timeBucketSize}min", "sum")`,
+        `summarize(collectd.localhost.docker-stats-${name}.counter-blockio-write, "${timeBucketSize}min", "sum")`,
+        `summarize(collectd.localhost.docker-stats-${name}.counter-network-read, "${timeBucketSize}min", "sum")`,
+        `summarize(collectd.localhost.docker-stats-${name}.counter-network-write, "${timeBucketSize}min", "sum")`,
+        `summarize(collectd.localhost.docker-stats-${name}.gauge-blockio-read, "${fromMinutes}min", "max")`,
+        `summarize(collectd.localhost.docker-stats-${name}.gauge-blockio-write, "${fromMinutes}min", "max")`,
+        `summarize(collectd.localhost.docker-stats-${name}.gauge-network-read, "${fromMinutes}min", "max")`,
+        `summarize(collectd.localhost.docker-stats-${name}.gauge-network-write, "${fromMinutes}min", "max")`,
+    ];
 
-    const [memoryError, memoryResponse] = await safe(superagent.get(GRAPHITE_RENDER_URL)
-        .query(memoryQuery)
-        .timeout(30 * 1000)
-        .ok(() => true));
+    const results = [];
 
-    if (memoryError) throw new BoxError(BoxError.NETWORK_ERROR, memoryError.message);
-    if (memoryResponse.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, `Unknown error: ${memoryResponse.status} ${memoryResponse.text}`);
-
-    let diskDataPoints;
-    if (app.manifest.addons.localstorage) {
-        const diskQuery = {
-            target: `summarize(collectd.localhost.du-${app.id}.capacity-usage, "${timeBucketSize}min", "avg")`,
+    for (const target of targets) {
+        const query = {
+            target: target,
             format: 'json',
             from: `-${fromMinutes}min`,
-            until: 'now'
+            until: 'now',
+            noNullPoints: !!noNullPoints
         };
-        if (noNullPoints) diskQuery.noNullPoints = true;
 
-        const [diskError, diskResponse] = await safe(superagent.get(GRAPHITE_RENDER_URL)
-            .query(diskQuery)
-            .timeout(30 * 1000)
-            .ok(() => true));
+        const [error, response] = await safe(superagent.get(graphiteUrl).query(query).timeout(30 * 1000).ok(() => true));
+        if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+        if (response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, `Unknown error with ${target}: ${response.status} ${response.text}`);
 
-        if (diskError) throw new BoxError(BoxError.NETWORK_ERROR, diskError.message);
-        if (diskResponse.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, `Unknown error: ${diskResponse.status} ${diskResponse.text}`);
-
-        // we may not have any datapoints
-        if (diskResponse.body.length === 0) diskDataPoints = [];
-        else diskDataPoints = diskResponse.body[0].datapoints;
-    } else {
-        diskDataPoints = [];
+        results.push(response.body[0] && response.body[0].datapoints ? response.body[0].datapoints : []);
     }
 
-    // app proxy instances have no container and thus no datapoints
-    return { memory: memoryResponse.body[0] || { datapoints: [] }, disk: { datapoints: diskDataPoints } };
+    // results are datapoints[[value, ts], [value, ts], ...];
+    return {
+        cpu: results[0],
+        memory: results[1],
+        blockRead: results[2],
+        blockWrite: results[3],
+        networkRead: results[4],
+        networkWrite: results[5],
+        blockReadTotal: results[6][0] && results[6][0][0] ? results[6][0][0] : 0,
+        blockWriteTotal: results[7][0] && results[7][0][0] ? results[7][0][0] : 0,
+        networkReadTotal: results[8][0] && results[8][0][0] ? results[8][0][0] : 0,
+        networkWriteTotal: results[9][0] && results[9][0][0] ? results[9][0][0] : 0,
+        cpuCount: os.cpus().length
+    };
 }
 
 async function getSystem(fromMinutes, noNullPoints) {
@@ -82,9 +88,10 @@ async function getSystem(fromMinutes, noNullPoints) {
     assert.strictEqual(typeof noNullPoints, 'boolean');
 
     const timeBucketSize = fromMinutes > (24 * 60) ? (6*60) : 5;
+    const graphiteUrl = await getGraphiteUrl();
 
-    const cpuQuery = `summarize(sum(collectd.localhost.aggregation-cpu-average.cpu-system, collectd.localhost.aggregation-cpu-average.cpu-user), "${timeBucketSize}min", "avg")`;
-    const memoryQuery = `summarize(sum(collectd.localhost.memory.memory-used, collectd.localhost.swap.swap-used), "${timeBucketSize}min", "avg")`;
+    const cpuQuery = `summarize(sum(collectd.localhost.aggregation-cpu-sum.cpu-system, collectd.localhost.aggregation-cpu-sum.cpu-user), "${timeBucketSize}min", "avg")`;
+    const memoryQuery = `summarize(collectd.localhost.memory.memory-used, "${timeBucketSize}min", "avg")`;
 
     const query = {
         target: [ cpuQuery, memoryQuery ],
@@ -93,92 +100,25 @@ async function getSystem(fromMinutes, noNullPoints) {
         until: 'now'
     };
 
-    const [memCpuError, memCpuResponse] = await safe(superagent.get(GRAPHITE_RENDER_URL)
-        .query(query)
-        .timeout(30 * 1000)
-        .ok(() => true));
-
+    const [memCpuError, memCpuResponse] = await safe(superagent.get(graphiteUrl).query(query).timeout(30 * 1000).ok(() => true));
     if (memCpuError) throw new BoxError(BoxError.NETWORK_ERROR, memCpuError.message);
     if (memCpuResponse.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, `Unknown error: ${memCpuResponse.status} ${memCpuResponse.text}`);
 
-    const allApps = await apps.list();
     const appResponses = {};
-    for (const app of allApps) {
-        appResponses[app.id] = await getByApp(app, fromMinutes, noNullPoints);
+    for (const app of await apps.list()) {
+        appResponses[app.id] = await getContainerStats(app.id, fromMinutes, noNullPoints);
     }
 
-    const diskInfo = await system.getDisks();
-
-    // segregate locations into the correct disks based on 'filesystem'
-    diskInfo.disks.forEach(function (disk, index) {
-        disk.id = index;
-        disk.contains = [];
-
-        if (disk.filesystem === diskInfo.platformDataDisk) disk.contains.push({ type: 'standard', label: 'Platform data', id: 'platformdata', usage: 0 });
-        if (disk.filesystem === diskInfo.boxDataDisk) disk.contains.push({ type: 'standard', label: 'Box data', id: 'boxdata', usage: 0 });
-        if (disk.filesystem === diskInfo.dockerDataDisk) disk.contains.push({ type: 'standard', label: 'Docker images', id: 'docker', usage: 0 });
-        if (disk.filesystem === diskInfo.mailDataDisk) disk.contains.push({ type: 'standard', label: 'Email data', id: 'maildata', usage: 0 });
-        if (disk.filesystem === diskInfo.backupsDisk) disk.contains.push({ type: 'standard', label: 'Backup data', id: 'cloudron-backup', usage: 0 });
-
-        // attach appIds which reside on this disk
-        const apps = Object.keys(diskInfo.apps).filter(function (appId) { return diskInfo.apps[appId] === disk.filesystem; });
-        apps.forEach(function (appId) {
-            disk.contains.push({ type: 'app', id: appId, label: '', usage: 0 });
-        });
-
-        // attach volumeIds which reside on this disk
-        const volumes = Object.keys(diskInfo.volumes).filter(function (volumeId) { return diskInfo.volumes[volumeId] === disk.filesystem; });
-        volumes.forEach(function (volumeId) {
-            disk.contains.push({ type: 'volume', id: volumeId, label: '', usage: 0 });
-        });
-    });
-
-    for (const disk of diskInfo.disks) {
-        // /dev/sda1 -> sda1
-        // /dev/mapper/foo.com -> mapper_foo_com (see #348)
-        let diskName = disk.filesystem.slice(disk.filesystem.indexOf('/', 1) + 1);
-        diskName = diskName.replace(/\/|\./g, '_');
-
-        const target = [
-            `absolute(collectd.localhost.df-${diskName}.df_complex-free)`,
-            `absolute(collectd.localhost.df-${diskName}.df_complex-reserved)`, // reserved for root (default: 5%) tune2fs -l/m
-            `absolute(collectd.localhost.df-${diskName}.df_complex-used)`
-        ];
-
-        const diskQuery = {
-            target: target,
-            format: 'json',
-            from: '-1day',
-            until: 'now'
-        };
-
-        const [diskError, diskResponse] = await safe(superagent.get(GRAPHITE_RENDER_URL).query(diskQuery).timeout(30 * 1000).ok(() => true));
-        if (diskError) throw new BoxError(BoxError.NETWORK_ERROR, diskError.message);
-        if (diskResponse.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, `Unknown error: ${diskResponse.status} ${diskResponse.text}`);
-
-        disk.size = diskResponse.body[2].datapoints[0][0] + diskResponse.body[1].datapoints[0][0] + diskResponse.body[0].datapoints[0][0];
-        disk.free = diskResponse.body[0].datapoints[0][0];
-        disk.occupied = diskResponse.body[2].datapoints[0][0];
-
-        for (const content of disk.contains) {
-            const query = {
-                target: `absolute(collectd.localhost.du-${content.id}.capacity-usage)`,
-                format: 'json',
-                from: '-1day',
-                until: 'now'
-            };
-
-            const [error, response] = await safe(superagent.get(GRAPHITE_RENDER_URL).query(query).timeout(30 * 1000).ok(() => true));
-            if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
-            if (response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, `Unknown error: ${response.status} ${response.text}`);
-
-            // we may not have any datapoints
-            if (response.body.length === 0) content.usage = null;
-            else content.usage = response.body[0].datapoints[0][0];
-
-            console.log(content)
-        }
+    const serviceResponses = {};
+    for (const serviceId of await services.listServices()) {
+        serviceResponses[serviceId] = await getContainerStats(serviceId, fromMinutes, noNullPoints);
     }
 
-    return { cpu: memCpuResponse.body[0], memory: memCpuResponse.body[1], apps: appResponses, disks: diskInfo.disks };
+    return {
+        cpu: memCpuResponse.body[0] && memCpuResponse.body[0].datapoints ? memCpuResponse.body[0].datapoints : [],
+        memory: memCpuResponse.body[1] && memCpuResponse.body[1].datapoints ? memCpuResponse.body[1].datapoints : [],
+        apps: appResponses,
+        services: serviceResponses,
+        cpuCount: os.cpus().length
+    };
 }

src/infra_version.js

@@ -6,7 +6,7 @@
 
 exports = module.exports = {
     // a version change recreates all containers with latest docker config
-    'version': '49.1.0',
+    'version': '49.2.0',
 
     'baseImages': [
         { repo: 'cloudron/base', tag: 'cloudron/base:3.2.0@sha256:ba1d566164a67c266782545ea9809dc611c4152e27686fd14060332dd88263ea' }
@@ -16,12 +16,12 @@ exports = module.exports = {
     // docker inspect --format='{{index .RepoDigests 0}}' $IMAGE to get the sha256
     'images': {
         'turn': { repo: 'cloudron/turn', tag: 'cloudron/turn:1.4.0@sha256:45817f1631992391d585f171498d257487d872480fd5646723a2b956cc4ef15d' },
-        'mysql': { repo: 'cloudron/mysql', tag: 'cloudron/mysql:3.2.1@sha256:75cef64ba4917ba9ec68bc0c9d9ba3a9eeae00a70173cd6d81cc6118038737d9' },
-        'postgresql': { repo: 'cloudron/postgresql', tag: 'cloudron/postgresql:4.3.1@sha256:b0c564d097b765d4a639330843e2e813d2c87fc8ed34b7df7550bf2c6df0012c' },
-        'mongodb': { repo: 'cloudron/mongodb', tag: 'cloudron/mongodb:4.2.1@sha256:f7f689beea07b1c6a9503a48f6fb38ef66e5b22f59fc585a92842a6578b33d46' },
-        'redis': { repo: 'cloudron/redis', tag: 'cloudron/redis:3.3.0@sha256:89c4e8083631b6d16b5d630d9b27f8ecf301c62f81219d77bd5948a1f4a4375c' },
-        'mail': { repo: 'cloudron/mail', tag: 'cloudron/mail:3.7.0@sha256:a41a52ba45bea0b2f14be82f8480d5f4583d806dc1f9c99c3bce858d2c9f27d7' },
-        'graphite': { repo: 'cloudron/graphite', tag: 'cloudron/graphite:3.1.0@sha256:30ec3a01964a1e01396acf265183997c3e17fb07eac1a82b979292cc7719ff4b' },
+        'mysql': { repo: 'cloudron/mysql', tag: 'cloudron/mysql:3.2.2@sha256:8648ca5a16fcdec72799b919c5f62419fd19e922e3d98d02896b921ae6127ef4' },
+        'postgresql': { repo: 'cloudron/postgresql', tag: 'cloudron/postgresql:4.3.4@sha256:84effb12e93d4e6467fedf3a426989980927ef90be61e73bde43476eebadf2a8' },
+        'mongodb': { repo: 'cloudron/mongodb', tag: 'cloudron/mongodb:4.2.2@sha256:df928d7dce1ac6454fc584787fa863f6d5e7ee0abb775dde5916a555fc94c3c7' },
+        'redis': { repo: 'cloudron/redis', tag: 'cloudron/redis:3.3.1@sha256:383e11a5c7a54d17eb6bbceb0ffa92f486167be6ea9978ec745c8c8e9b7dfb19' },
+        'mail': { repo: 'cloudron/mail', tag: 'cloudron/mail:3.7.0@sha256:44df70c8030cb9da452568c32fae7cae447e3b98cf48fdbc7b27a2466e706473' },
+        'graphite': { repo: 'cloudron/graphite', tag: 'cloudron/graphite:3.1.0@sha256:182e5cae69fbddc703cb9f91be909452065c7ae159e9836cc88317c7a00f0e62' },
         'sftp': { repo: 'cloudron/sftp', tag: 'cloudron/sftp:3.6.1@sha256:ba4b9a1fe274c0ef0a900e5d0deeb8f3da08e118798d1d90fbf995cc0cf6e3a3' }
     }
 };

src/mail.js

@@ -516,7 +516,11 @@ const RBL_LIST = [
 
 // this function currently only looks for black lists based on IP. TODO: also look up by domain
 async function checkRblStatus(domain) {
-    const ip = await sysinfo.getServerIPv4();
+    const [error, ip] = await safe(sysinfo.getServerIPv4());
+    if (error) {
+        debug(`checkRblStatus: unable to determine server IPv4: ${error.message}`);
+        return { status: false, ip: null, servers: [] };
+    }
 
     const flippedIp = ip.split('.').reverse().join('.');
 

src/mounts.js

@@ -184,7 +184,7 @@ async function getStatus(mountType, hostPath) {
 
             if (end !== -1) message = lines.slice(start, end+1).map(line => line['MESSAGE']).join('\n');
         }
-        if (!message) message = `Could not determine failure reason. ${safe.error ? safe.error.message : ''}`;
+        if (!message) message = `Could not determine mount failure reason. ${safe.error ? safe.error.message : ''}`;
     } else {
         message = 'Mounted';
     }

src/nginxconfig.ejs

@@ -247,10 +247,10 @@ server {
         client_max_body_size 0;
     }
 
-    # graphite paths (uncomment block below and visit /graphite-web/dashboard)
+    # graphite paths (uncomment block below and visit /graphite-web/)
     # remember to comment out the CSP policy as well to access the graphite dashboard
     # location ~ ^/graphite-web/ {
-    #     proxy_pass   http://127.0.0.1:8417;
+    #     proxy_pass http://172.18.0.6:8000;
     #     client_max_body_size 1m;
     # }
 

src/paths.js

@@ -31,7 +31,6 @@ exports = module.exports = {
     ACME_CHALLENGES_DIR: path.join(baseDir(), 'platformdata/acme'),
     ADDON_CONFIG_DIR: path.join(baseDir(), 'platformdata/addons'),
     MAIL_CONFIG_DIR: path.join(baseDir(), 'platformdata/addons/mail'),
-    COLLECTD_APPCONFIG_DIR: path.join(baseDir(), 'platformdata/collectd/collectd.conf.d'),
     LOGROTATE_CONFIG_DIR: path.join(baseDir(), 'platformdata/logrotate.d'),
     NGINX_CONFIG_DIR: path.join(baseDir(), 'platformdata/nginx'),
     NGINX_APPCONFIG_DIR: path.join(baseDir(), 'platformdata/nginx/applications'),
@@ -39,6 +38,7 @@ exports = module.exports = {
     BACKUP_INFO_DIR: path.join(baseDir(), 'platformdata/backup'),
     UPDATE_DIR: path.join(baseDir(), 'platformdata/update'),
     UPDATE_CHECKER_FILE: path.join(baseDir(), 'platformdata/update/updatechecker.json'),
+    DISK_USAGE_FILE: path.join(baseDir(), 'platformdata/diskusage.json'),
     SNAPSHOT_INFO_FILE: path.join(baseDir(), 'platformdata/backup/snapshot-info.json'),
     DYNDNS_INFO_FILE: path.join(baseDir(), 'platformdata/dyndns-info.json'),
     DHPARAMS_FILE: path.join(baseDir(), 'platformdata/dhparams.pem'),

src/routes/applinks.js

@@ -47,6 +47,7 @@ async function get(req, res, next) {
 
     const [error, result] = await safe(applinks.get(req.params.id));
     if (error) return next(BoxError.toHttpError(error));
+    if (!result) return next(new HttpError(404, 'Applink not found'));
 
     // we have a separate route for this
     delete result.icon;

src/routes/apps.js

@@ -58,6 +58,7 @@ exports = module.exports = {
     updateBackup,
 
     getLimits,
+    getGraphs,
 
     load
 };
@@ -68,6 +69,7 @@ const apps = require('../apps.js'),
     BoxError = require('../boxerror.js'),
     constants = require('../constants.js'),
     debug = require('debug')('box:routes/apps'),
+    graphs = require('../graphs.js'),
     HttpError = require('connect-lastmile').HttpError,
     HttpSuccess = require('connect-lastmile').HttpSuccess,
     safe = require('safetydance'),
@@ -497,14 +499,14 @@ async function importApp(req, res, next) {
     const data = req.body;
 
     if ('remotePath' in data) { // if not provided, we import in-place
-        if (typeof data.remotePath !== 'string') return next(new HttpError(400, 'remotePath must be string'));
+        if (typeof data.remotePath !== 'string' || !data.remotePath) return next(new HttpError(400, 'remotePath must be non-empty string'));
         if (typeof data.backupFormat !== 'string') return next(new HttpError(400, 'backupFormat must be string'));
 
         if ('backupConfig' in data && typeof data.backupConfig !== 'object') return next(new HttpError(400, 'backupConfig must be an object'));
 
         const backupConfig = req.body.backupConfig;
 
-        if (req.body.backupConfig) {
+        if (backupConfig) {
             if (typeof backupConfig.provider !== 'string') return next(new HttpError(400, 'provider is required'));
             if ('password' in backupConfig && typeof backupConfig.password !== 'string') return next(new HttpError(400, 'password must be a string'));
             if ('encryptedFilenames' in backupConfig && typeof backupConfig.encryptedFilenames !== 'boolean') return next(new HttpError(400, 'encryptedFilenames must be a boolean'));
@@ -959,3 +961,16 @@ async function getLimits(req, res, next) {
 
     next(new HttpSuccess(200, { limits }));
 }
+
+async function getGraphs(req, res, next) {
+    assert.strictEqual(typeof req.app, 'object');
+
+    if (!req.query.fromMinutes || !parseInt(req.query.fromMinutes)) return next(new HttpError(400, 'fromMinutes must be a number'));
+
+    const fromMinutes = parseInt(req.query.fromMinutes);
+    const noNullPoints = !!req.query.noNullPoints;
+    const [error, result] = await safe(graphs.getContainerStats(req.app.id, fromMinutes, noNullPoints));
+    if (error) return next(new HttpError(500, error));
+
+    next(new HttpSuccess(200, result));
+}

src/routes/cloudron.js

@@ -10,6 +10,8 @@ exports = module.exports = {
     isRebootRequired,
     getConfig,
     getDisks,
+    getDiskUsage,
+    updateDiskUsage,
     getMemory,
     getUpdateInfo,
     update,
@@ -23,7 +25,8 @@ exports = module.exports = {
     getServerIpv6,
     getLanguages,
     syncExternalLdap,
-    syncDnsRecords
+    syncDnsRecords,
+    getSystemGraphs
 };
 
 const assert = require('assert'),
@@ -34,6 +37,7 @@ const assert = require('assert'),
     debug = require('debug')('box:routes/cloudron'),
     eventlog = require('../eventlog.js'),
     externalLdap = require('../externalldap.js'),
+    graphs = require('../graphs.js'),
     HttpError = require('connect-lastmile').HttpError,
     HttpSuccess = require('connect-lastmile').HttpSuccess,
     safe = require('safetydance'),
@@ -166,7 +170,21 @@ async function getDisks(req, res, next) {
     const [error, result] = await safe(system.getDisks());
     if (error) return next(BoxError.toHttpError(error));
 
-    next(new HttpSuccess(200, result));
+    next(new HttpSuccess(200, { disks: result }));
+}
+
+async function getDiskUsage(req, res, next) {
+    const [error, result] = await safe(system.getDiskUsage());
+    if (error) return next(BoxError.toHttpError(error));
+
+    next(new HttpSuccess(200, { usage: result }));
+}
+
+async function updateDiskUsage(req, res, next) {
+    const [error, taskId] = await safe(cloudron.updateDiskUsage());
+    if (error) return next(BoxError.toHttpError(error));
+
+    next(new HttpSuccess(201, { taskId }));
 }
 
 async function getMemory(req, res, next) {
@@ -327,3 +345,14 @@ async function syncDnsRecords(req, res, next) {
 
     next(new HttpSuccess(201, { taskId }));
 }
+
+async function getSystemGraphs(req, res, next) {
+    if (!req.query.fromMinutes || !parseInt(req.query.fromMinutes)) return next(new HttpError(400, 'fromMinutes must be a number'));
+
+    const fromMinutes = parseInt(req.query.fromMinutes);
+    const noNullPoints = !!req.query.noNullPoints;
+    const [error, result] = await safe(graphs.getSystem(fromMinutes, noNullPoints));
+    if (error) return next(new HttpError(500, error));
+
+    next(new HttpSuccess(200, result));
+}

src/routes/graphs.js

@@ -1,36 +0,0 @@
-'use strict';
-
-exports = module.exports = {
-    getSystemGraphs,
-    getAppGraphs
-};
-
-const assert = require('assert'),
-    graphs = require('../graphs.js'),
-    HttpError = require('connect-lastmile').HttpError,
-    HttpSuccess = require('connect-lastmile').HttpSuccess,
-    safe = require('safetydance');
-
-async function getSystemGraphs(req, res, next) {
-    if (!req.query.fromMinutes || !parseInt(req.query.fromMinutes)) return next(new HttpError(400, 'fromMinutes must be a number'));
-
-    const fromMinutes = parseInt(req.query.fromMinutes);
-    const noNullPoints = !!req.query.noNullPoints;
-    const [error, result] = await safe(graphs.getSystem(fromMinutes, noNullPoints));
-    if (error) return next(new HttpError(500, error));
-
-    next(new HttpSuccess(200, result));
-}
-
-async function getAppGraphs(req, res, next) {
-    assert.strictEqual(typeof req.app, 'object');
-
-    if (!req.query.fromMinutes || !parseInt(req.query.fromMinutes)) return next(new HttpError(400, 'fromMinutes must be a number'));
-
-    const fromMinutes = parseInt(req.query.fromMinutes);
-    const noNullPoints = !!req.query.noNullPoints;
-    const [error, result] = await safe(graphs.getByApp(req.app, fromMinutes, noNullPoints));
-    if (error) return next(new HttpError(500, error));
-
-    next(new HttpSuccess(200, result));
-}

src/routes/index.js

@@ -12,7 +12,6 @@ exports = module.exports = {
     domains: require('./domains.js'),
     eventlog: require('./eventlog.js'),
     filemanager: require('./filemanager.js'),
-    graphs: require('./graphs.js'),
     groups: require('./groups.js'),
     mail: require('./mail.js'),
     mailserver: require('./mailserver.js'),

src/routes/services.js

@@ -7,12 +7,14 @@ exports = module.exports = {
     getLogs,
     getLogStream,
     restart,
-    rebuild
+    rebuild,
+    getGraphs
 };
 
 const assert = require('assert'),
     AuditSource = require('../auditsource.js'),
     BoxError = require('../boxerror.js'),
+    graphs = require('../graphs.js'),
     HttpError = require('connect-lastmile').HttpError,
     HttpSuccess = require('connect-lastmile').HttpSuccess,
     safe = require('safetydance'),
@@ -131,3 +133,16 @@ async function rebuild(req, res, next) {
 
     next(new HttpSuccess(202, {}));
 }
+
+async function getGraphs(req, res, next) {
+    assert.strictEqual(typeof req.params.service, 'string');
+
+    if (!req.query.fromMinutes || !parseInt(req.query.fromMinutes)) return next(new HttpError(400, 'fromMinutes must be a number'));
+
+    const fromMinutes = parseInt(req.query.fromMinutes);
+    const noNullPoints = !!req.query.noNullPoints;
+    const [error, result] = await safe(graphs.getContainerStats(req.params.service, fromMinutes, noNullPoints));
+    if (error) return next(new HttpError(500, error));
+
+    next(new HttpSuccess(200, result));
+}

src/routes/test/cloudron-test.js

@@ -10,11 +10,13 @@ const constants = require('../../constants.js'),
     expect = require('expect.js'),
     http = require('http'),
     os = require('os'),
+    paths = require('../../paths.js'),
+    safe = require('safetydance'),
     superagent = require('superagent'),
     settings = require('../../settings.js');
 
 describe('Cloudron API', function () {
-    const { setup, cleanup, serverUrl, owner, user } = common;
+    const { setup, cleanup, serverUrl, owner, user, waitForTask } = common;
 
     before(setup);
     after(cleanup);
@@ -369,6 +371,58 @@ describe('Cloudron API', function () {
         });
     });
 
+    describe('disks', function () {
+        it('succeeds', async function () {
+            const response = await superagent.get(`${serverUrl}/api/v1/cloudron/disks`)
+                .query({ access_token: owner.token });
+
+            expect(response.statusCode).to.equal(200);
+            expect(response.body.disks).to.be.ok();
+            expect(Object.keys(response.body.disks).some(fs => response.body.disks[fs].mountpoint === '/')).to.be(true);
+        });
+    });
+
+    describe('disk usage', function () {
+        it('get succeeds with no cache', async function () {
+            safe.fs.unlinkSync(paths.DISK_USAGE_FILE);
+
+            const response = await superagent.get(`${serverUrl}/api/v1/cloudron/disk_usage`)
+                .query({ access_token: owner.token })
+                .send({});
+
+            expect(response.statusCode).to.equal(200);
+            expect(response.body).to.eql({ usage: null });
+        });
+
+        it('update the cache', async function () {
+            const response = await superagent.post(`${serverUrl}/api/v1/cloudron/disk_usage`)
+                .query({ access_token: owner.token });
+
+            expect(response.statusCode).to.equal(201);
+            expect(response.body.taskId).to.be.ok();
+            await waitForTask(response.body.taskId);
+        });
+
+        it('get succeeds with cache', async function () {
+            const response = await superagent.get(`${serverUrl}/api/v1/cloudron/disk_usage`)
+                .query({ access_token: owner.token })
+                .send({});
+
+            expect(response.statusCode).to.equal(200);
+            expect(response.body.usage.ts).to.be.a('number');
+
+            const filesystems = Object.keys(response.body.usage.disks);
+            let dockerUsage = null;
+            for (const fs of filesystems) {
+                for (const content of response.body.usage.disks[fs].contents) {
+                    if (content.id === 'docker') dockerUsage = content;
+                }
+            }
+            expect(dockerUsage).to.be.ok();
+            expect(dockerUsage.usage).to.be.a('number');
+        });
+    });
+
     describe('languages', function () {
         it('succeeds', async function () {
             const response = await superagent.get(`${serverUrl}/api/v1/cloudron/languages`);

src/scripts/backupupload.js

@@ -56,8 +56,8 @@ function dumpMemoryInfo() {
         return (bytes / Math.pow(1024, i)).toFixed(2) * 1 + '' + sizes[i];
     }
 
-    debug(`process: rss: ${h(mu.rss)} heapUsed: ${h(mu.heapUsed)} heapTotal: ${h(mu.heapTotal)} external: ${h(mu.external)}`);
-    debug(`v8 heap: used ${h(hs.used_heap_size)} total: ${h(hs.total_heap_size)} max: ${h(hs.heap_size_limit)}`);
+    debug(`process: rss=${h(mu.rss)} heapUsed=${h(mu.heapUsed)} heapTotal=${h(mu.heapTotal)} external=${h(mu.external)}`
+        + ` v8 heap: used=${h(hs.used_heap_size)} total=${h(hs.total_heap_size)} max=${h(hs.heap_size_limit)}`);
 }
 
 (async function main() {
@@ -65,7 +65,7 @@ function dumpMemoryInfo() {
     await settings.initCache();
 
     dumpMemoryInfo();
-    const timerId = setInterval(dumpMemoryInfo, 30000);
+    const timerId = setInterval(dumpMemoryInfo, 180 * 1000);
 
     const [uploadError] = await safe(backuptask.upload(remotePath, format, dataLayoutString, throttledProgressCallback(5000)));
     debug('upload completed. error: ', uploadError);

src/scripts/configurecollectd.sh

@@ -1,44 +0,0 @@
-#!/bin/bash
-
-set -eu -o pipefail
-
-if [[ ${EUID} -ne 0 ]]; then
-    echo "This script should be run as root." > /dev/stderr
-    exit 1
-fi
-
-if [[ $# == 1 && "$1" == "--check" ]]; then
-    echo "OK"
-    exit 0
-fi
-
-cmd="$1"
-metric="$2" # note that this can also be 'cloudron-backup' or appid
-
-if [[ "${BOX_ENV}" == "cloudron" ]]; then
-    # when restoring the cloudron with many apps, the apptasks rush in to restart
-    # collectd which makes systemd/collectd very unhappy and puts the collectd in
-    # inactive state
-    for i in {1..10}; do
-        echo "Restarting collectd"
-        if systemctl restart collectd; then
-            break
-        fi
-        echo "Failed to reload collectd. Maybe some other apptask is restarting it"
-        sleep $((RANDOM%30))
-    done
-
-    # delete old stats when uninstalling an app
-    if [[ "${cmd}" == "remove" ]]; then
-        echo "Removing collectd stats of ${metric}"
-
-        for i in {1..10}; do
-            if rm -rf ${HOME}/platformdata/graphite/whisper/collectd/localhost/*${metric}*; then
-                break
-            fi
-            echo "Failed to remove collectd directory. collectd possibly generated data in the middle of removal"
-            sleep 3
-        done
-    fi
-fi
-

src/scripts/du.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -eu -o pipefail
+
+if [[ ${EUID} -ne 0 ]]; then
+    echo "This script should be run as root." > /dev/stderr
+    exit 1
+fi
+
+if [[ $# -eq 0 ]]; then
+    echo "No arguments supplied"
+    exit 1
+fi
+
+if [[ "$1" == "--check" ]]; then
+    echo "OK"
+    exit 0
+fi
+
+path="$1"
+
+# -B1 makes du print block sizes and not apparent sizes (to match df which also uses block sizes)
+du -DsB1 "${path}"

src/server.js

@@ -116,8 +116,10 @@ function initializeExpressSync() {
     router.post('/api/v1/cloudron/check_for_updates',        json, token, authorizeAdmin, routes.cloudron.checkForUpdates);
     router.get ('/api/v1/cloudron/reboot',                         token, authorizeAdmin, routes.cloudron.isRebootRequired);
     router.post('/api/v1/cloudron/reboot',                   json, token, authorizeAdmin, routes.cloudron.reboot);
-    router.get ('/api/v1/cloudron/graphs',                         token, authorizeAdmin, routes.graphs.getSystemGraphs);
+    router.get ('/api/v1/cloudron/graphs',                         token, authorizeAdmin, routes.cloudron.getSystemGraphs);
     router.get ('/api/v1/cloudron/disks',                          token, authorizeAdmin, routes.cloudron.getDisks);
+    router.get ('/api/v1/cloudron/disk_usage',                     token, authorizeAdmin, routes.cloudron.getDiskUsage);
+    router.post('/api/v1/cloudron/disk_usage',                     token, authorizeAdmin, routes.cloudron.updateDiskUsage);
     router.get ('/api/v1/cloudron/memory',                         token, authorizeAdmin, routes.cloudron.getMemory);
     router.get ('/api/v1/cloudron/logs/:unit',                     token, authorizeAdmin, routes.cloudron.getLogs);
     router.get ('/api/v1/cloudron/logstream/:unit',                token, authorizeAdmin, routes.cloudron.getLogStream);
@@ -248,7 +250,7 @@ function initializeExpressSync() {
     router.get ('/api/v1/apps/:id/eventlog',                           token, routes.apps.load, authorizeOperator, routes.apps.listEventlog);
     router.get ('/api/v1/apps/:id/limits',                             token, routes.apps.load, authorizeOperator, routes.apps.getLimits);
     router.get ('/api/v1/apps/:id/task',                               token, routes.apps.load, authorizeOperator, routes.apps.getTask);
-    router.get ('/api/v1/apps/:id/graphs',                             token, routes.apps.load, authorizeOperator, routes.graphs.getAppGraphs);
+    router.get ('/api/v1/apps/:id/graphs',                             token, routes.apps.load, authorizeOperator, routes.apps.getGraphs);
     router.post('/api/v1/apps/:id/clone',                        json, token, routes.apps.load, authorizeAdmin, routes.apps.clone);
     router.get ('/api/v1/apps/:id/download',                           token, routes.apps.load, authorizeOperator, routes.apps.downloadFile);
     router.post('/api/v1/apps/:id/upload',                       json, token, multipart, routes.apps.load, authorizeOperator, routes.apps.uploadFile);
@@ -328,9 +330,9 @@ function initializeExpressSync() {
     router.del ('/api/v1/mail/:domain/lists/:name',                   token, authorizeMailManager, routes.mail.delList);
 
     // support routes
-    router.post('/api/v1/support/ticket',         json, token, authorizeAdmin, routes.support.canCreateTicket, routes.support.createTicket);
-    router.get ('/api/v1/support/remote_support',       token, authorizeAdmin, routes.support.getRemoteSupport);
-    router.post('/api/v1/support/remote_support', json, token, authorizeAdmin, routes.support.canEnableRemoteSupport, routes.support.enableRemoteSupport);
+    router.post('/api/v1/support/ticket',         json, token, authorizeOwner, routes.support.canCreateTicket, routes.support.createTicket);
+    router.get ('/api/v1/support/remote_support',       token, authorizeOwner, routes.support.getRemoteSupport);
+    router.post('/api/v1/support/remote_support', json, token, authorizeOwner, routes.support.canEnableRemoteSupport, routes.support.enableRemoteSupport);
 
     // domain routes
     router.post('/api/v1/domains',                    json, token, authorizeAdmin, routes.domains.add);
@@ -354,6 +356,7 @@ function initializeExpressSync() {
     router.get ('/api/v1/services',                          token, authorizeAdmin, routes.services.list);
     router.get ('/api/v1/services/:service',                 token, authorizeAdmin, routes.services.get);
     router.post('/api/v1/services/:service',           json, token, authorizeAdmin, routes.services.configure);
+    router.get ('/api/v1/services/:service/graphs',          token, authorizeAdmin, routes.services.getGraphs);
     router.get ('/api/v1/services/:service/logs',            token, authorizeAdmin, routes.services.getLogs);
     router.get ('/api/v1/services/:service/logstream',       token, authorizeAdmin, routes.services.getLogStream);
     router.post('/api/v1/services/:service/restart',   json, token, authorizeAdmin, routes.services.restart);

src/services.js

@@ -310,7 +310,7 @@ async function containerStatus(containerName, tokenEnvName) {
 }
 
 async function listServices() {
-    let serviceIds = Object.keys(SERVICES);
+    const serviceIds = Object.keys(SERVICES);
 
     const result = await apps.list();
     for (let app of result) {
@@ -1646,6 +1646,7 @@ async function startGraphite(existingInfra) {
     const readOnly = !serviceConfig.recoveryMode ? '--read-only' : '';
     const cmd = serviceConfig.recoveryMode ? '/bin/bash -c \'echo "Debug mode. Sleeping" && sleep infinity\'' : '';
 
+    // port 2003 is used by collectd
     const runCmd = `docker run --restart=always -d --name="graphite" \
         --hostname graphite \
         --net cloudron \
@@ -1659,8 +1660,6 @@ async function startGraphite(existingInfra) {
         --dns 172.18.0.1 \
         --dns-search=. \
         -p 127.0.0.1:2003:2003 \
-        -p 127.0.0.1:2004:2004 \
-        -p 127.0.0.1:8417:8000 \
         -v "${paths.PLATFORM_DATA_DIR}/graphite:/var/lib/graphite" \
         --label isCloudronManaged=true \
         ${readOnly} -v /tmp -v /run "${tag}" ${cmd}`;
@@ -1883,7 +1882,10 @@ async function statusGraphite() {
     if (error && error.reason === BoxError.NOT_FOUND) return { status: exports.SERVICE_STATUS_STOPPED };
     if (error) throw error;
 
-    const [networkError, response] = await safe(superagent.get('http://127.0.0.1:8417/graphite-web/dashboard')
+    const ip = safe.query(container, 'NetworkSettings.Networks.cloudron.IPAddress', null);
+    if (!ip) throw new BoxError(BoxError.INACTIVE, 'Error getting IP of graphite service');
+
+    const [networkError, response] = await safe(superagent.get(`http://${ip}:8000/graphite-web/dashboard`)
         .timeout(20000)
         .ok(() => true));
 

src/settings.js

@@ -468,8 +468,6 @@ async function setBackupConfig(backupConfig) {
     }
 
     notifyChange(exports.BACKUP_CONFIG_KEY, backupConfig);
-
-    await backups.configureCollectd(backupConfig);
 }
 
 async function setBackupCredentials(credentials) {
@@ -487,8 +485,6 @@ async function setBackupCredentials(credentials) {
     await set(exports.BACKUP_CONFIG_KEY, JSON.stringify(backupConfig));
 
     notifyChange(exports.BACKUP_CONFIG_KEY, backupConfig);
-
-    await backups.configureCollectd(backupConfig);
 }
 
 async function getServicesConfig() {

src/storage/filesystem.js

@@ -1,8 +1,9 @@
 'use strict';
 
 exports = module.exports = {
-    getRootPath,
-    checkPreconditions,
+    getBackupRootPath,
+    getBackupProviderStatus,
+    getAvailableSize,
 
     upload,
     download,
@@ -33,7 +34,6 @@ const PROVIDER_EXT4 = 'ext4';
 const assert = require('assert'),
     BoxError = require('../boxerror.js'),
     constants = require('../constants.js'),
-    DataLayout = require('../datalayout.js'),
     debug = require('debug')('box:storage/filesystem'),
     df = require('@sindresorhus/df'),
     fs = require('fs'),
@@ -45,7 +45,7 @@ const assert = require('assert'),
     shell = require('../shell.js');
 
 // storage api
-function getRootPath(apiConfig) {
+function getBackupRootPath(apiConfig) {
     assert.strictEqual(typeof apiConfig, 'object');
 
     switch (apiConfig.provider) {
@@ -62,43 +62,26 @@ function getRootPath(apiConfig) {
     }
 }
 
-// binary units (non SI) 1024 based
-function prettyBytes(bytes) {
-    assert.strictEqual(typeof bytes, 'number');
+async function getBackupProviderStatus(apiConfig) {
+    assert.strictEqual(typeof apiConfig, 'object');
 
-    const i = Math.floor(Math.log(bytes) / Math.log(1024)),
-        sizes = ['B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'];
+    // Check filesystem is mounted so we don't write into the actual folder on disk
+    if (mounts.isManagedProvider(apiConfig.provider) || apiConfig.provider === 'mountpoint') {
+        const hostPath = mounts.isManagedProvider(apiConfig.provider) ? paths.MANAGED_BACKUP_MOUNT_DIR : apiConfig.mountPoint;
+        return await mounts.getStatus(apiConfig.provider, hostPath); // { state, message }
+    }
 
-    return (bytes / Math.pow(1024, i)).toFixed(2) * 1 + '' + sizes[i];
+    return await mounts.getStatus(apiConfig.provider, apiConfig.backupFolder);
 }
 
 // the du call in the function below requires root
-async function checkPreconditions(apiConfig, dataLayout) {
+async function getAvailableSize(apiConfig) {
     assert.strictEqual(typeof apiConfig, 'object');
-    assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
 
-    let used = 0;
-    for (let localPath of dataLayout.localPaths()) {
-        debug(`checkPreconditions: getting disk usage of ${localPath}`);
-        let result = safe.child_process.execSync(`du -Dsb ${localPath}`, { encoding: 'utf8' });
-        if (!result) throw new BoxError(BoxError.FS_ERROR, `du error: ${safe.error.message}`);
-        used += parseInt(result, 10);
-    }
-
-    debug(`checkPreconditions: ${used} bytes`);
-
-    const [error, result] = await safe(df.file(getRootPath(apiConfig)));
+    const [error, dfResult] = await safe(df.file(getBackupRootPath(apiConfig)));
     if (error) throw new BoxError(BoxError.FS_ERROR, `Error when checking for disk space: ${error.message}`);
 
-    // Check filesystem is mounted so we don't write into the actual folder on disk
-    if (apiConfig.provider === PROVIDER_SSHFS || apiConfig.provider === PROVIDER_CIFS || apiConfig.provider === PROVIDER_NFS || apiConfig.provider === PROVIDER_EXT4 || apiConfig.provider === PROVIDER_XFS) {
-        if (result.mountpoint !== paths.MANAGED_BACKUP_MOUNT_DIR) throw new BoxError(BoxError.FS_ERROR, 'Backup target is not mounted');
-    } else if (apiConfig.provider === PROVIDER_MOUNTPOINT) {
-        if (result.mountpoint === '/') throw new BoxError(BoxError.FS_ERROR, `${apiConfig.backupFolder} is not mounted`);
-    }
-
-    const needed = 0.6 * used + (1024 * 1024 * 1024); // check if there is atleast 1GB left afterwards. aim for 60% because rsync/tgz won't need full 100%
-    if (result.available <= needed) throw new BoxError(BoxError.FS_ERROR, `Not enough disk space for backup. Needed: ${prettyBytes(needed)} Available: ${prettyBytes(result.available)}`);
+    return dfResult.available;
 }
 
 function hasChownSupportSync(apiConfig) {
@@ -308,7 +291,7 @@ async function testConfig(apiConfig) {
         if (!safe.child_process.execSync(`mountpoint -q -- ${apiConfig.mountPoint}`)) throw new BoxError(BoxError.BAD_FIELD, `${apiConfig.mountPoint} is not mounted`);
     }
 
-    const basePath = getRootPath(apiConfig);
+    const basePath = getBackupRootPath(apiConfig);
     const field = apiConfig.provider === PROVIDER_FILESYSTEM ? 'backupFolder' : 'mountPoint';
 
     if (!safe.fs.mkdirSync(path.join(basePath, 'snapshot'), { recursive: true }) && safe.error.code !== 'EEXIST') {

src/storage/gcs.js

@@ -1,8 +1,9 @@
 'use strict';
 
 exports = module.exports = {
-    getRootPath,
-    checkPreconditions,
+    getBackupRootPath,
+    getBackupProviderStatus,
+    getAvailableSize,
 
     upload,
     exists,
@@ -29,7 +30,6 @@ const assert = require('assert'),
     async = require('async'),
     BoxError = require('../boxerror.js'),
     constants = require('../constants.js'),
-    DataLayout = require('../datalayout.js'),
     debug = require('debug')('box:storage/gcs'),
     PassThrough = require('stream').PassThrough,
     path = require('path'),
@@ -66,15 +66,22 @@ function getBucket(apiConfig) {
 }
 
 // storage api
-function getRootPath(apiConfig) {
+function getBackupRootPath(apiConfig) {
     assert.strictEqual(typeof apiConfig, 'object');
 
     return apiConfig.prefix;
 }
 
-async function checkPreconditions(apiConfig, dataLayout) {
+async function getBackupProviderStatus(apiConfig) {
     assert.strictEqual(typeof apiConfig, 'object');
-    assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
+
+    return { state: 'active' };
+}
+
+async function getAvailableSize(apiConfig) {
+    assert.strictEqual(typeof apiConfig, 'object');
+
+    return Number.POSITIVE_INFINITY;
 }
 
 function upload(apiConfig, backupFilePath, sourceStream, callback) {

src/storage/interface.js

@@ -11,8 +11,9 @@
 //     for the other API calls we leave it to the backend to retry. this allows
 //     them to tune the concurrency based on failures/rate limits accordingly
 exports = module.exports = {
-    getRootPath,
-    checkPreconditions,
+    getBackupRootPath,
+    getBackupProviderStatus,
+    getAvailableSize,
 
     upload,
 
@@ -34,8 +35,7 @@ exports = module.exports = {
 };
 
 const assert = require('assert'),
-    BoxError = require('../boxerror.js'),
-    DataLayout = require('../datalayout.js');
+    BoxError = require('../boxerror.js');
 
 function removePrivateFields(apiConfig) {
     // in-place removal of tokens and api keys with constants.SECRET_PLACEHOLDER
@@ -47,16 +47,23 @@ function injectPrivateFields(newConfig, currentConfig) {
     // in-place injection of tokens and api keys which came in with constants.SECRET_PLACEHOLDER
 }
 
-function getRootPath(apiConfig) {
+function getBackupRootPath(apiConfig) {
     assert.strictEqual(typeof apiConfig, 'object');
 
     // Result: path at the backup storage
     return '/';
 }
 
-async function checkPreconditions(apiConfig, dataLayout) {
+async function getBackupProviderStatus(apiConfig) {
     assert.strictEqual(typeof apiConfig, 'object');
-    assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
+
+    return { state: 'active' };
+}
+
+async function getAvailableSize(apiConfig) {
+    assert.strictEqual(typeof apiConfig, 'object');
+
+    return Number.POSITIVE_INFINITY;
 }
 
 function upload(apiConfig, backupFilePath, sourceStream, callback) {

src/storage/noop.js

@@ -1,8 +1,9 @@
 'use strict';
 
 exports = module.exports = {
-    getRootPath,
-    checkPreconditions,
+    getBackupRootPath,
+    getBackupProviderStatus,
+    getAvailableSize,
 
     upload,
     exists,
@@ -23,17 +24,23 @@ exports = module.exports = {
 
 const assert = require('assert'),
     BoxError = require('../boxerror.js'),
-    DataLayout = require('../datalayout.js'),
     debug = require('debug')('box:storage/noop');
 
-function getRootPath(apiConfig) {
+function getBackupRootPath(apiConfig) {
     assert.strictEqual(typeof apiConfig, 'object');
     return '';
 }
 
-async function checkPreconditions(apiConfig, dataLayout) {
+async function getBackupProviderStatus(apiConfig) {
     assert.strictEqual(typeof apiConfig, 'object');
-    assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
+
+    return { state: 'active' };
+}
+
+async function getAvailableSize(apiConfig) {
+    assert.strictEqual(typeof apiConfig, 'object');
+
+    return Number.POSITIVE_INFINITY;
 }
 
 function upload(apiConfig, backupFilePath, sourceStream, callback) {

src/storage/s3.js

@@ -1,8 +1,9 @@
 'use strict';
 
 exports = module.exports = {
-    getRootPath,
-    checkPreconditions,
+    getBackupRootPath,
+    getBackupProviderStatus,
+    getAvailableSize,
 
     upload,
     exists,
@@ -31,7 +32,6 @@ const assert = require('assert'),
     AwsSdk = require('aws-sdk'),
     BoxError = require('../boxerror.js'),
     constants = require('../constants.js'),
-    DataLayout = require('../datalayout.js'),
     debug = require('debug')('box:storage/s3'),
     https = require('https'),
     path = require('path'),
@@ -92,15 +92,22 @@ function getS3Config(apiConfig) {
 }
 
 // storage api
-function getRootPath(apiConfig) {
+function getBackupRootPath(apiConfig) {
     assert.strictEqual(typeof apiConfig, 'object');
 
     return apiConfig.prefix;
 }
 
-async function checkPreconditions(apiConfig, dataLayout) {
+async function getBackupProviderStatus(apiConfig) {
     assert.strictEqual(typeof apiConfig, 'object');
-    assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
+
+    return { state: 'active' };
+}
+
+async function getAvailableSize(apiConfig) {
+    assert.strictEqual(typeof apiConfig, 'object');
+
+    return Number.POSITIVE_INFINITY;
 }
 
 function upload(apiConfig, backupFilePath, sourceStream, callback) {

src/system.js

@@ -4,101 +4,97 @@ exports = module.exports = {
     getDisks,
     checkDiskSpace,
     getMemory,
-    getMemoryAllocation
+    getMemoryAllocation,
+    getDiskUsage,
+    updateDiskUsage
 };
 
 const apps = require('./apps.js'),
     assert = require('assert'),
     BoxError = require('./boxerror.js'),
-    constants = require('./constants.js'),
     debug = require('debug')('box:disks'),
     df = require('@sindresorhus/df'),
     docker = require('./docker.js'),
     notifications = require('./notifications.js'),
     os = require('os'),
+    path = require('path'),
     paths = require('./paths.js'),
     safe = require('safetydance'),
     settings = require('./settings.js'),
+    shell = require('./shell.js'),
     volumes = require('./volumes.js');
 
-async function getVolumeDisks(appsDataDisk) {
-    assert.strictEqual(typeof appsDataDisk, 'string');
+const DU_CMD = path.join(__dirname, 'scripts/du.sh');
 
-    let volumeDisks = {};
-    const allVolumes = await volumes.list();
+async function du(file) {
+    assert.strictEqual(typeof file, 'string');
 
-    for (const volume of allVolumes) {
-        const [error, result] = await safe(df(volume.hostPath));
-        volumeDisks[volume.id] = error ? appsDataDisk : result.filesystem; // ignore any errors
-    }
+    const [error, stdoutResult] = await safe(shell.promises.sudo('system', [ DU_CMD, file ], {}));
+    if (error) throw new BoxError(BoxError.FS_ERROR, error);
 
-    return volumeDisks;
-}
-
-async function getAppDisks(appsDataDisk) {
-    assert.strictEqual(typeof appsDataDisk, 'string');
-
-    let appDisks = {};
-
-    const allApps = await apps.list();
-    for (const app of allApps) {
-        if (app.manifest.id === constants.PROXY_APP_APPSTORE_ID) continue;
-
-        if (!app.storageVolumeId) {
-            appDisks[app.id] = appsDataDisk;
-        } else {
-            const dataDir = await apps.getStorageDir(app);
-            const [error, result] = await safe(df.file(dataDir));
-            appDisks[app.id] = error ? appsDataDisk : result.filesystem; // ignore any errors
-        }
-    }
-
-    return appDisks;
-}
-
-async function getBackupsFilesystem() {
-    const backupConfig = await settings.getBackupConfig();
-
-    if (backupConfig.provider !== 'filesystem') return null;
-
-    const result = await df.file(backupConfig.backupFolder);
-    return result.filesystem;
+    return parseInt(stdoutResult.trim(), 10);
 }
 
 async function getDisks() {
-    const info = await docker.info();
+    let [dfError, dfEntries] = await safe(df());
+    if (dfError) throw new BoxError(BoxError.FS_ERROR, `Error running df: ${dfError.message}`);
 
-    let [error, allDisks] = await safe(df());
-    if (error) throw new BoxError(BoxError.FS_ERROR, error);
+    const disks = {}; // by file system
+    let rootDisk;
 
-    // filter by ext4 and then sort to make sure root disk is first
-    const ext4Disks = allDisks.filter((r) => r.type === 'ext4' || r.type === 'xfs').sort((a, b) => a.mountpoint.localeCompare(b.mountpoint));
-
-    const diskInfos = [];
-    for (const p of [ paths.BOX_DATA_DIR, paths.MAIL_DATA_DIR, paths.PLATFORM_DATA_DIR, paths.APPS_DATA_DIR, info.DockerRootDir ]) {
-        const [dfError, diskInfo] = await safe(df.file(p));
-        if (dfError) throw new BoxError(BoxError.FS_ERROR, dfError);
-        diskInfos.push(diskInfo);
+    for (const disk of dfEntries) {
+        if (disk.type !== 'ext4' && disk.type !== 'xfs') continue;
+        if (disk.mountpoint === '/') rootDisk = disk;
+        disks[disk.filesystem] = {
+            filesystem: disk.filesystem,
+            type: disk.type,
+            size: disk.size,
+            used: disk.used,
+            available: disk.available,
+            capacity: disk.capacity,
+            mountpoint: disk.mountpoint,
+            contents: [] // filled below
+        };
     }
 
-    const backupsFilesystem = await getBackupsFilesystem();
+    const standardPaths = [
+        { type: 'standard', id: 'platformdata', path: paths.PLATFORM_DATA_DIR },
+        { type: 'standard', id: 'boxdata',      path: paths.BOX_DATA_DIR },
+        { type: 'standard', id: 'maildata',     path: paths.MAIL_DATA_DIR },
+    ];
 
-    const result = {
-        disks: ext4Disks, // root disk is first. { filesystem, type, size, used, available, capacity, mountpoint }
-        boxDataDisk: diskInfos[0].filesystem,
-        mailDataDisk: diskInfos[1].filesystem,
-        platformDataDisk: diskInfos[2].filesystem,
-        appsDataDisk: diskInfos[3].filesystem,
-        dockerDataDisk: diskInfos[4].filesystem,
-        backupsDisk: backupsFilesystem,
-        apps: {}, // filled below
-        volumes: {} // filled below
-    };
+    for (const stdPath of standardPaths) {
+        const [dfError, diskInfo] = await safe(df.file(stdPath.path));
+        if (dfError) throw new BoxError(BoxError.FS_ERROR, `Error getting std path: ${dfError.message}`);
+        disks[diskInfo.filesystem].contents.push(stdPath);
+    }
 
-    result.apps = await getAppDisks(result.appsDataDisk);
-    result.volumes = await getVolumeDisks(result.appsDataDisk);
+    const backupConfig = await settings.getBackupConfig();
+    if (backupConfig.provider === 'filesystem') {
+        const [, dfResult] = await safe(df.file(backupConfig.backupFolder));
+        disks[dfResult?.filesystem || rootDisk.filesystem].contents.push({ type: 'standard', id: 'cloudron-backup', path: backupConfig.backupFolder });
+    }
 
-    return result;
+    const [dockerError, dockerInfo] = await safe(docker.info());
+    if (!dockerError) {
+        const [, dfResult] = await safe(df.file(dockerInfo.DockerRootDir));
+        disks[dfResult?.filesystem || rootDisk.filesystem].contents.push({ type: 'standard', id: 'docker', path: dockerInfo.DockerRootDir });
+    }
+
+    for (const volume of await volumes.list()) {
+        const [, dfResult] = await safe(df(volume.hostPath));
+        disks[dfResult?.filesystem || rootDisk.filesystem].contents.push({ type: 'volume', id: volume.id, path: volume.hostPath });
+    }
+
+    for (const app of await apps.list()) {
+        if (!app.manifest.addons?.localstorage) continue;
+
+        const dataDir = await apps.getStorageDir(app);
+        const [, dfResult] = await safe(df.file(dataDir));
+        disks[dfResult?.filesystem || rootDisk.filesystem].contents.push({ type: 'app', id: app.id, path: dataDir });
+    }
+
+    return disks;
 }
 
 async function checkDiskSpace() {
@@ -108,18 +104,14 @@ async function checkDiskSpace() {
 
     let markdownMessage = '';
 
-    disks.disks.forEach(function (entry) {
-        // ignore other filesystems but where box, app and platform data is
-        if (entry.filesystem !== disks.boxDataDisk
-            && entry.filesystem !== disks.platformDataDisk
-            && entry.filesystem !== disks.appsDataDisk
-            && entry.filesystem !== disks.backupsDisk
-            && entry.filesystem !== disks.dockerDataDisk) return false;
+    for (const filesystem of Object.keys(disks)) {
+        const disk = disks[filesystem];
+        if (disk.contents.length === 0) continue; // ignore if nothing interesting here
 
-        if (entry.available <= (1.25 * 1024 * 1024 * 1024)) { // 1.5G
-            markdownMessage += `* ${entry.filesystem} is at ${entry.capacity*100}% capacity.\n`;
+        if (disk.available <= (1.25 * 1024 * 1024 * 1024)) { // 1.5G
+            markdownMessage += `* ${disk.filesystem} is at ${disk.capacity*100}% capacity.\n`;
         }
-    });
+    }
 
     debug(`checkDiskSpace: disk space checked. out of space: ${markdownMessage || 'no'}`);
 
@@ -152,3 +144,40 @@ function getMemoryAllocation(limit) {
 
     return Math.round(Math.round(limit * ratio) / 1048576) * 1048576; // nearest MB
 }
+
+async function getDiskUsage() {
+    return safe.JSON.parse(safe.fs.readFileSync(paths.DISK_USAGE_FILE, 'utf8'));
+}
+
+async function updateDiskUsage(progressCallback) {
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    const disks = await getDisks();
+    const filesystems = Object.keys(disks);
+    const now = Date.now();
+
+    let percent = 1;
+
+    for (const filesystem of filesystems) {
+        const disk = disks[filesystem];
+
+        percent += (100/filesystems.length);
+        progressCallback({ percent, message: `Checking contents of ${filesystem}`});
+
+        for (const content of disk.contents) {
+            progressCallback({ message: `Checking du of ${JSON.stringify(content)}`});
+            if (content.id === 'docker') {
+                content.usage = (await docker.df()).LayersSize;
+            } else {
+                const [error, usage] = await safe(du(content.path));
+                if (error) progressCallback({ message: `du error: ${error.message}`}); // can happen if app is installing etc
+                content.usage = usage || 0;
+            }
+            progressCallback({ message: `du of ${JSON.stringify(content)}: ${content.usage}`});
+        }
+    }
+
+    if (!safe.fs.writeFileSync(paths.DISK_USAGE_FILE, JSON.stringify({ ts: now, disks }), 'utf8')) throw new BoxError(BoxError.FS_ERROR, `Could not write du cache file: ${safe.error.message}`);
+
+    return disks;
+}

src/tasks.js

@@ -28,6 +28,7 @@ exports = module.exports = {
     TASK_SYNC_EXTERNAL_LDAP: 'syncExternalLdap',
     TASK_CHANGE_MAIL_LOCATION: 'changeMailLocation',
     TASK_SYNC_DNS_RECORDS: 'syncDnsRecords',
+    TASK_UPDATE_DISK_USAGE: 'updateDiskUsage',
 
     // error codes
     ESTOPPED: 'stopped',

src/taskworker.js

@@ -15,6 +15,7 @@ const apptask = require('./apptask.js'),
     reverseProxy = require('./reverseproxy.js'),
     safe = require('safetydance'),
     settings = require('./settings.js'),
+    system = require('./system.js'),
     tasks = require('./tasks.js'),
     updater = require('./updater.js');
 
@@ -28,6 +29,7 @@ const TASKS = { // indexed by task type
     syncExternalLdap: externalLdap.sync,
     changeMailLocation: mail.changeLocation,
     syncDnsRecords: dns.syncDnsRecords,
+    updateDiskUsage: system.updateDiskUsage,
 
     _identity: async (arg, progressCallback) => { progressCallback(); return arg; },
     _error: async (arg, progressCallback) => { progressCallback(); throw new Error(`Failed for arg: ${arg}`); },

src/test/applinks-test.js

@@ -39,9 +39,6 @@ describe('Applinks', function () {
 
     it('can add applink with redirect', async function () {
         APPLINK_0.id = await applinks.add(JSON.parse(JSON.stringify(APPLINK_0)));
-
-        // redirect should have put www in
-        APPLINK_0.upstreamUri = 'https://www.cloudron.io/';
     });
 
     it('can add second applink with attributes', async function () {
@@ -52,7 +49,7 @@ describe('Applinks', function () {
         APPLINK_2.id = await applinks.add(JSON.parse(JSON.stringify(APPLINK_2)));
 
         const result = await applinks.get(APPLINK_2.id);
-        expect(result.upstreamUri).to.eql('https://www.google.com/');
+        expect(result.upstreamUri).to.eql(APPLINK_2.upstreamUri); // should not have changed
         expect(result.icon.length).to.not.eql(0);
     });
 
@@ -70,18 +67,17 @@ describe('Applinks', function () {
         expect(result.length).to.equal(4);
         expect(result[1].id).to.eql(APPLINK_0.id);
         expect(result[1].upstreamUri).to.eql(APPLINK_0.upstreamUri);
-        expect(result[2].id).to.eql(APPLINK_1.id);
-        expect(result[2].upstreamUri).to.eql(APPLINK_1.upstreamUri);
-        expect(result[2].label).to.eql(APPLINK_1.label);
-        expect(result[2].accessRestriction).to.eql(APPLINK_1.accessRestriction);
-        expect(result[2].tags).to.eql(APPLINK_1.tags);
-        expect(result[2].icon.toString('base64')).to.eql(APPLINK_1.icon);
+        expect(result[3].id).to.eql(APPLINK_1.id);
+        expect(result[3].upstreamUri).to.eql(APPLINK_1.upstreamUri);
+        expect(result[3].label).to.eql(APPLINK_1.label);
+        expect(result[3].accessRestriction).to.eql(APPLINK_1.accessRestriction);
+        expect(result[3].tags).to.eql(APPLINK_1.tags);
+        expect(result[3].icon.toString('base64')).to.eql(APPLINK_1.icon);
     });
 
     it('cannot get applink with wrong id', async function () {
-        const [error] = await safe(applinks.get('doesnotexist'));
-        expect(error).to.be.a(BoxError);
-        expect(error.reason).to.eql(BoxError.NOT_FOUND);
+        const result = await applinks.get('doesnotexist');
+        expect(result).to.be(null);
     });
 
     it('can get applink', async function () {
@@ -125,8 +121,7 @@ describe('Applinks', function () {
     it('can remove applink', async function () {
         await applinks.remove(APPLINK_0.id);
 
-        const [error] = await safe(applinks.get(APPLINK_0.id));
-        expect(error).to.be.a(BoxError);
-        expect(error.reason).to.eql(BoxError.NOT_FOUND);
+        const result = await applinks.get(APPLINK_0.id);
+        expect(result).to.be(null);
     });
 });

src/test/check-install

@@ -18,7 +18,7 @@ scripts=("${SOURCE_DIR}/src/scripts/clearvolume.sh" \
          "${SOURCE_DIR}/src/scripts/restartservice.sh" \
          "${SOURCE_DIR}/src/scripts/update.sh" \
          "${SOURCE_DIR}/src/scripts/collectlogs.sh" \
-         "${SOURCE_DIR}/src/scripts/configurecollectd.sh" \
+         "${SOURCE_DIR}/src/scripts/du.sh" \
          "${SOURCE_DIR}/src/scripts/remotesupport.sh" \
          "${SOURCE_DIR}/src/scripts/starttask.sh" \
          "${SOURCE_DIR}/src/scripts/stoptask.sh" \

src/test/docker-test.js

@@ -0,0 +1,23 @@
+/* jslint node:true */
+/* global it:false */
+/* global before:false */
+/* global after:false */
+/* global describe:false */
+
+'use strict';
+
+const common = require('./common.js'),
+    docker = require('../docker.js'),
+    expect = require('expect.js');
+
+describe('docker', function () {
+    const { setup, cleanup } = common;
+
+    before(setup);
+    after(cleanup);
+
+    it('can df', async function () {
+        const output = await docker.df();
+        expect(output).to.be.ok();
+    });
+});

src/test/system-test.js

@@ -36,5 +36,9 @@ describe('System', function () {
         expect(memory.memory).to.be.a('number');
         expect(memory.swap).to.be.a('number');
     });
-});
 
+    it('can get diskUsage', async function () {
+        const usage = await system.getDiskUsage();
+        expect(usage).to.be(null); // nothing cached
+    });
+});

src/volumes.js

@@ -14,13 +14,10 @@ exports = module.exports = {
 
 const assert = require('assert'),
     BoxError = require('./boxerror.js'),
-    collectd = require('./collectd.js'),
     constants = require('./constants.js'),
     database = require('./database.js'),
     debug = require('debug')('box:volumes'),
-    ejs = require('ejs'),
     eventlog = require('./eventlog.js'),
-    fs = require('fs'),
     mounts = require('./mounts.js'),
     path = require('path'),
     paths = require('./paths.js'),
@@ -30,8 +27,6 @@ const assert = require('assert'),
 
 const VOLUMES_FIELDS = [ 'id', 'name', 'hostPath', 'creationTime', 'mountType', 'mountOptionsJson' ].join(',');
 
-const COLLECTD_CONFIG_EJS = fs.readFileSync(__dirname + '/collectd/volume.ejs', { encoding: 'utf8' });
-
 function postProcess(result) {
     assert.strictEqual(typeof result, 'object');
 
@@ -104,9 +99,6 @@ async function add(volume, auditSource) {
     // in theory, we only need to do this mountpoint volumes. but for some reason a restart is required to detect new "mounts"
     safe(services.rebuildService('sftp', auditSource), { debug });
 
-    const collectdConf = ejs.render(COLLECTD_CONFIG_EJS, { volumeId: id, hostPath: volume.hostPath });
-    await collectd.addProfile(id, collectdConf);
-
     return id;
 }
 
@@ -170,8 +162,6 @@ async function del(volume, auditSource) {
     } else {
         await safe(mounts.removeMount(volume));
     }
-
-    await collectd.removeProfile(volume.id);
 }
 
 async function mountAll() {