graphite: carbon crash fix

https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=923464
https://forum.cloudron.io/topic/4797/graphite-keeps-crashing-oom/34
(cherry picked from commit cd300bb6e2)

CHANGES

@@ -2238,3 +2238,18 @@
 * sftp: fix rebuild condition
 * Fix display of user management/dashboard visiblity for email apps
 * graphite: disable tagdb and reduce log noise
+
+[6.2.6]
+* Fix issue where collectd is restarted too quickly before graphite
+
+[6.2.7]
+* collectd: restart on graphite upgrade
+
+[6.2.8]
+* linode object storage: update aws sdk to make it work again
+* Fix crash in blocklist setting when source and list have mixed ip versions
+* mysql: bump connection limit to 200
+* namecheap: fix issue where DNS updates and del were not working
+* turn: turn off verbose logging
+* Fix crash when parsing df output (set LC_ALL for box service)
+

package-lock.json

@@ -54,9 +54,9 @@
       "integrity": "sha512-d4VSA86eL/AFTe5xtyZX+ePUjE8dIFu2T8zmdeNBSa5/kNgXPCx/o/wbFNHAGLJdGnk1vddRuMESD9HbOC8irw=="
     },
     "@google-cloud/storage": {
-      "version": "5.8.0",
-      "resolved": "https://registry.npmjs.org/@google-cloud/storage/-/storage-5.8.0.tgz",
-      "integrity": "sha512-WOShvBPOfkDXUzXMO+3j8Bzus+PFI9r1Ey9dLG2Zf458/PVuFTtaRWntd9ZiDG8g90zl2LmnA1JkDCreGUKr5g==",
+      "version": "5.8.3",
+      "resolved": "https://registry.npmjs.org/@google-cloud/storage/-/storage-5.8.3.tgz",
+      "integrity": "sha512-g++NTmpmwbZZEnBhJi3y1D3YyZ2Y+1xL5blp96eeJhffginMym5tRw/AGNZblDI35U2K1FTJEYqIZ31tbEzs8w==",
       "requires": {
         "@google-cloud/common": "^3.6.0",
         "@google-cloud/paginator": "^3.0.0",
@@ -103,9 +103,9 @@
           "integrity": "sha512-A1B3Bh1UmL0bidM/YX2NsCOTnGJePL9rO/M+Mw3m9f2gUpfokS0hi5Eah0WSUEWZdZhIZtMjkIYS7mDfOqNHbg=="
         },
         "google-auth-library": {
-          "version": "7.0.2",
-          "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-7.0.2.tgz",
-          "integrity": "sha512-vjyNZR3pDLC0u7GHLfj+Hw9tGprrJwoMwkYGqURCXYITjCrP9HprOyxVV+KekdLgATtWGuDkQG2MTh0qpUPUgg==",
+          "version": "7.0.3",
+          "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-7.0.3.tgz",
+          "integrity": "sha512-6wJNYqY1QUr5I2lWaUkkzOT2b9OCNhNQrdFOt/bsBbGb7T7NCdEvrBsXraUm+KTUGk2xGlQ7m9RgUd4Llcw8NQ==",
           "requires": {
             "arrify": "^2.0.0",
             "base64-js": "^1.3.0",
@@ -316,9 +316,9 @@
       "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k="
     },
     "aws-sdk": {
-      "version": "2.850.0",
-      "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.850.0.tgz",
-      "integrity": "sha512-RqPeSKe1JlhTUL9+xUsp771ZtMY7JICoQEnFJuJ+JVqGyILhg95L4t8S5KnznUfWYc0pcpTiHKLmPteHyHS3pw==",
+      "version": "2.879.0",
+      "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.879.0.tgz",
+      "integrity": "sha512-HRfjGwST1U9AvCJFAyqpAJwbjFR4LqUyEUk77qdJpdYHL9pGPHdnEfGRkBkPn36xcC7Em6gVvFveVoEihbQUyQ==",
       "requires": {
         "buffer": "4.9.2",
         "events": "1.1.1",
@@ -359,7 +359,7 @@
     },
     "backoff": {
       "version": "2.5.0",
-      "resolved": "https://registry.npmjs.org/backoff/-/backoff-2.5.0.tgz",
+      "resolved": false,
       "integrity": "sha1-9hbtqdPktmuMp/ynn2lXIsX44m8=",
       "requires": {
         "precond": "0.2"
@@ -574,6 +574,17 @@
         "is-glob": "~4.0.1",
         "normalize-path": "~3.0.0",
         "readdirp": "~3.5.0"
+      },
+      "dependencies": {
+        "readdirp": {
+          "version": "3.5.0",
+          "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.5.0.tgz",
+          "integrity": "sha512-cMhu7c/8rdhkHXWsY+osBhfSy0JikwpHK/5+imo+LpeasTF8ouErHrlYkwT0++njiyuDvc7OFY5T3ukvZ8qmFQ==",
+          "dev": true,
+          "requires": {
+            "picomatch": "^2.2.1"
+          }
+        }
       }
     },
     "chownr": {
@@ -1761,9 +1772,9 @@
       },
       "dependencies": {
         "google-auth-library": {
-          "version": "7.0.2",
-          "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-7.0.2.tgz",
-          "integrity": "sha512-vjyNZR3pDLC0u7GHLfj+Hw9tGprrJwoMwkYGqURCXYITjCrP9HprOyxVV+KekdLgATtWGuDkQG2MTh0qpUPUgg==",
+          "version": "7.0.3",
+          "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-7.0.3.tgz",
+          "integrity": "sha512-6wJNYqY1QUr5I2lWaUkkzOT2b9OCNhNQrdFOt/bsBbGb7T7NCdEvrBsXraUm+KTUGk2xGlQ7m9RgUd4Llcw8NQ==",
           "requires": {
             "arrify": "^2.0.0",
             "base64-js": "^1.3.0",
@@ -1843,9 +1854,9 @@
       }
     },
     "glob-parent": {
-      "version": "5.1.1",
-      "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.1.tgz",
-      "integrity": "sha512-FnI+VGOpnlGHWZxthPGR+QhR78fuiK0sNLkHQv+bL9fQi57lNNdquIbna/WrfROrolq8GK5Ek6BiMwqL/voRYQ==",
+      "version": "5.1.2",
+      "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz",
+      "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==",
       "dev": true,
       "requires": {
         "is-glob": "^4.0.1"
@@ -2732,9 +2743,9 @@
       "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A=="
     },
     "mocha": {
-      "version": "8.3.0",
-      "resolved": "https://registry.npmjs.org/mocha/-/mocha-8.3.0.tgz",
-      "integrity": "sha512-TQqyC89V1J/Vxx0DhJIXlq9gbbL9XFNdeLQ1+JsnZsVaSOV1z3tWfw0qZmQJGQRIfkvZcs7snQnZnOCKoldq1Q==",
+      "version": "8.3.2",
+      "resolved": "https://registry.npmjs.org/mocha/-/mocha-8.3.2.tgz",
+      "integrity": "sha512-UdmISwr/5w+uXLPKspgoV7/RXZwKRTiTjJ2/AC5ZiEztIoOYdfKb19+9jNmEInzx5pBsCyJQzarAxqIGBNYJhg==",
       "dev": true,
       "requires": {
         "@ungap/promise-all-settled": "1.1.2",
@@ -2873,9 +2884,9 @@
           "dev": true
         },
         "string-width": {
-          "version": "4.2.0",
-          "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.0.tgz",
-          "integrity": "sha512-zUz5JD+tgqtuDjMhwIg5uFVV3dtqZ9yQJlZVfq4I01/K5Paj5UHj7VyrQOJvzawSVlKpObApbfD0Ed6yJc+1eg==",
+          "version": "4.2.2",
+          "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.2.tgz",
+          "integrity": "sha512-XBJbT3N4JhVumXE0eoLU9DCjcaF92KLNqTmFCnG1pf8duUxFGwtP6AD6nkjw9a3IdiRtL3E2w3JDiE/xi3vOeA==",
           "dev": true,
           "requires": {
             "emoji-regex": "^8.0.0",
@@ -3112,9 +3123,9 @@
       "integrity": "sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw=="
     },
     "nock": {
-      "version": "13.0.7",
-      "resolved": "https://registry.npmjs.org/nock/-/nock-13.0.7.tgz",
-      "integrity": "sha512-WBz73VYIjdbO6BwmXODRQLtn7B5tldA9pNpWJe5QTtTEscQlY5KXU4srnGzBOK2fWakkXj69gfTnXGzmrsaRWw==",
+      "version": "13.0.11",
+      "resolved": "https://registry.npmjs.org/nock/-/nock-13.0.11.tgz",
+      "integrity": "sha512-sKZltNkkWblkqqPAsjYW0bm3s9DcHRPiMOyKO/PkfJ+ANHZ2+LA2PLe22r4lLrKgXaiSaDQwW3qGsJFtIpQIeQ==",
       "dev": true,
       "requires": {
         "debug": "^4.1.0",
@@ -3223,9 +3234,9 @@
       }
     },
     "nodemailer": {
-      "version": "6.4.18",
-      "resolved": "https://registry.npmjs.org/nodemailer/-/nodemailer-6.4.18.tgz",
-      "integrity": "sha512-ht9cXxQ+lTC+t00vkSIpKHIyM4aXIsQ1tcbQCn5IOnxYHi81W2XOaU66EQBFFpbtzLEBTC94gmkbD4mGZQzVpA=="
+      "version": "6.5.0",
+      "resolved": "https://registry.npmjs.org/nodemailer/-/nodemailer-6.5.0.tgz",
+      "integrity": "sha512-Tm4RPrrIZbnqDKAvX+/4M+zovEReiKlEXWDzG4iwtpL9X34MJY+D5LnQPH/+eghe8DLlAVshHAJZAZWBGhkguw=="
     },
     "nodemailer-fetch": {
       "version": "1.6.0",
@@ -3487,7 +3498,7 @@
     },
     "precond": {
       "version": "0.2.3",
-      "resolved": "https://registry.npmjs.org/precond/-/precond-0.2.3.tgz",
+      "resolved": false,
       "integrity": "sha1-qpWRvKokkj8eD0hJ0kD0fvwQdaw="
     },
     "pretty-bytes": {
@@ -3741,9 +3752,9 @@
       }
     },
     "readdirp": {
-      "version": "3.5.0",
-      "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.5.0.tgz",
-      "integrity": "sha512-cMhu7c/8rdhkHXWsY+osBhfSy0JikwpHK/5+imo+LpeasTF8ouErHrlYkwT0++njiyuDvc7OFY5T3ukvZ8qmFQ==",
+      "version": "3.6.0",
+      "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz",
+      "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==",
       "requires": {
         "picomatch": "^2.2.1"
       }
@@ -3947,9 +3958,9 @@
       }
     },
     "semver": {
-      "version": "7.3.4",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.4.tgz",
-      "integrity": "sha512-tCfb2WLjqFAtXn4KEdxIhalnRtoKFN7nAwj0B3ZXCbQloV2tq5eDbcTmT68JJD3nRJq24/XgxtQKFIpQdtvmVw==",
+      "version": "7.3.5",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.5.tgz",
+      "integrity": "sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ==",
       "requires": {
         "lru-cache": "^6.0.0"
       },
@@ -4660,9 +4671,9 @@
       }
     },
     "underscore": {
-      "version": "1.12.0",
-      "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.12.0.tgz",
-      "integrity": "sha512-21rQzss/XPMjolTiIezSu3JAjgagXKROtNrYFEOWK109qY1Uv2tVjPTZ1ci2HgvQDA16gHYSthQIJfB+XId/rQ=="
+      "version": "1.12.1",
+      "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.12.1.tgz",
+      "integrity": "sha512-hEQt0+ZLDVUMhebKxL4x1BTtDY7bavVofhZ9KZ4aI26X9SRaE+Y3m83XUL1UP2jn8ynjndwCCpEHdUG+9pP1Tw=="
     },
     "unique-string": {
       "version": "2.0.0",
@@ -4916,9 +4927,9 @@
       }
     },
     "ws": {
-      "version": "7.4.3",
-      "resolved": "https://registry.npmjs.org/ws/-/ws-7.4.3.tgz",
-      "integrity": "sha512-hr6vCR76GsossIRsr8OLR9acVVm1jyfEWvhbNjtgPOrfvAlKzvyeg/P6r8RuDjRyrcQoPQT7K0DGEPc7Ae6jzA=="
+      "version": "7.4.4",
+      "resolved": "https://registry.npmjs.org/ws/-/ws-7.4.4.tgz",
+      "integrity": "sha512-Qm8k8ojNQIMx7S+Zp8u/uHOx7Qazv3Yv4q68MiWWWOJhiwG5W3x7iqmRtJo8xxrciZUY4vRxUTJCKuRnF28ZZw=="
     },
     "xdg-basedir": {
       "version": "4.0.0",

package.json

@@ -12,10 +12,10 @@
   },
   "dependencies": {
     "@google-cloud/dns": "^2.1.0",
-    "@google-cloud/storage": "^5.8.0",
+    "@google-cloud/storage": "^5.8.3",
     "@sindresorhus/df": "git+https://github.com/cloudron-io/df.git#type",
     "async": "^3.2.0",
-    "aws-sdk": "^2.850.0",
+    "aws-sdk": "^2.879.0",
     "basic-auth": "^2.0.1",
     "body-parser": "^1.19.0",
     "cloudron-manifestformat": "^5.10.1",
@@ -46,19 +46,19 @@
     "multiparty": "^4.2.2",
     "mustache-express": "^1.3.0",
     "mysql": "^2.18.1",
-    "nodemailer": "^6.4.18",
+    "nodemailer": "^6.5.0",
     "nodemailer-smtp-transport": "^2.7.4",
     "once": "^1.4.0",
     "pretty-bytes": "^5.6.0",
     "progress-stream": "^2.0.0",
     "proxy-middleware": "^0.15.0",
     "qrcode": "^1.4.4",
-    "readdirp": "^3.5.0",
+    "readdirp": "^3.6.0",
     "request": "^2.88.2",
     "rimraf": "^3.0.2",
     "s3-block-read-stream": "^0.5.0",
     "safetydance": "^1.1.1",
-    "semver": "^7.3.4",
+    "semver": "^7.3.5",
     "showdown": "^1.9.1",
     "speakeasy": "^2.0.0",
     "split": "^1.0.1",
@@ -67,19 +67,19 @@
     "tar-fs": "github:cloudron-io/tar-fs#ignore_stat_error",
     "tar-stream": "^2.2.0",
     "tldjs": "^2.3.1",
-    "underscore": "^1.12.0",
+    "underscore": "^1.12.1",
     "uuid": "^8.3.2",
     "validator": "^13.5.2",
-    "ws": "^7.4.3",
+    "ws": "^7.4.4",
     "xml2js": "^0.4.23"
   },
   "devDependencies": {
     "expect.js": "*",
     "hock": "^1.4.1",
     "js2xmlparser": "^4.0.1",
-    "mocha": "^8.3.0",
+    "mocha": "^8.3.2",
     "mock-aws-s3": "git+https://github.com/cloudron-io/mock-aws-s3.git",
-    "nock": "^13.0.7",
+    "nock": "^13.0.11",
     "node-sass": "^5.0.0",
     "recursive-readdir": "^2.2.2"
   },

scripts/cloudron-support

@@ -39,10 +39,11 @@ while true; do
     --owner-login)
         admin_username=$(mysql -NB -uroot -ppassword -e "SELECT username FROM box.users WHERE role='owner' AND username IS NOT NULL ORDER BY createdAt LIMIT 1" 2>/dev/null)
         admin_password=$(pwgen -1s 12)
+        dashboard_domain=$(mysql -NB -uroot -ppassword -e "SELECT value FROM box.settings WHERE name='admin_fqdn'" 2>/dev/nul)
         ghost_file=/home/yellowtent/platformdata/cloudron_ghost.json
         printf '{"%s":"%s"}\n' "${admin_username}" "${admin_password}" > "${ghost_file}"
         chown yellowtent:yellowtent "${ghost_file}" && chmod o-r,g-r "${ghost_file}"
-        echo "Login as ${admin_username} / ${admin_password} . This password may only be used once. ${ghost_file} will be automatically removed after use."
+        echo "Login at https://${dashboard_domain} as ${admin_username} / ${admin_password} . This password may only be used once. ${ghost_file} will be automatically removed after use."
         exit 0
         ;;
     --) break;;

setup/start/collectd/collectd.conf

@@ -164,7 +164,9 @@ LoadPlugin swap
 #LoadPlugin vmem
 #LoadPlugin vserver
 #LoadPlugin wireless
-LoadPlugin write_graphite
+<LoadPlugin write_graphite>
+   FlushInterval 20
+</LoadPlugin>
 #LoadPlugin write_http
 #LoadPlugin write_riemann
 

setup/start/collectd/du.py

@@ -6,19 +6,26 @@ PATHS = [] # { name, dir, exclude }
 # there is a pattern in carbon/storage-schemas.conf which stores values every 12h for a year
 INTERVAL = 60 * 60 * 12 # twice a day. change values in docker-graphite if you change this
 
+# we used to pass the INTERVAL as a parameter to register_read. however, collectd write_graphite
+# takes a bit to load (tcp connection) and drops the du data. this then means that we have to wait
+# for INTERVAL secs for du data. instead, we just cache the value for INTERVAL instead
+CACHE = dict()
+CACHE_TIME = 0
+
 def du(pathinfo):
     # -B1 makes du print block sizes and not apparent sizes (to match df which also uses block sizes)
-    cmd = 'timeout 1800 du -DsB1 "{}"'.format(pathinfo['dir'])
+    dirname = pathinfo['dir']
+    cmd = 'timeout 1800 du -DsB1 "{}"'.format(dirname)
     if pathinfo['exclude'] != '':
         cmd += ' --exclude "{}"'.format(pathinfo['exclude'])
 
     collectd.info('computing size with command: %s' % cmd);
     try:
         size = subprocess.check_output(cmd, shell=True).split()[0].decode('utf-8')
-        collectd.info('\tsize of %s is %s (time: %i)' % (pathinfo['dir'], size, int(time.time())))
+        collectd.info('\tsize of %s is %s (time: %i)' % (dirname, size, int(time.time())))
         return size
     except Exception as e:
-        collectd.info('\terror getting the size of %s: %s' % (pathinfo['dir'], str(e)))
+        collectd.info('\terror getting the size of %s: %s' % (dirname, str(e)))
         return 0
 
 def parseSize(size):
@@ -64,19 +71,35 @@ def init():
     collectd.info('custom du plugin initialized with %s %s' % (PATHS, sys.version))
 
 def read():
+    global CACHE, CACHE_TIME
+
+    # read from cache if < 12 hours
+    read_cache = (time.time() - CACHE_TIME) < INTERVAL
+
+    if not read_cache:
+        CACHE_TIME = time.time()
+
     for pathinfo in PATHS:
-        size = du(pathinfo)
+        dirname = pathinfo['dir']
+        if read_cache and dirname in CACHE:
+            size = CACHE[dirname]
+        else:
+            size = du(pathinfo)
+            CACHE[dirname] = size
 
         # type comes from https://github.com/collectd/collectd/blob/master/src/types.db
         val = collectd.Values(type='capacity', plugin='du', plugin_instance=pathinfo['name'])
         val.dispatch(values=[size], type_instance='usage')
 
-    size = dockerSize()
+    if read_cache and 'docker' in CACHE:
+        size = CACHE['docker']
+    else:
+        size = dockerSize()
+        CACHE['docker'] = size
+
     val = collectd.Values(type='capacity', plugin='du', plugin_instance='docker')
     val.dispatch(values=[size], type_instance='usage')
 
-
-
 collectd.register_init(init)
 collectd.register_config(configure)
-collectd.register_read(read, INTERVAL)
+collectd.register_read(read)

setup/start/mysql.cnf

@@ -4,7 +4,7 @@
 # http://bugs.mysql.com/bug.php?id=68514
 [mysqld]
 performance_schema=OFF
-max_connections=50
+max_connections=200
 # on ec2, without this we get a sporadic connection drop when doing the initial migration
 max_allowed_packet=64M
 

setup/start/systemd/box.service

@@ -13,7 +13,8 @@ Type=idle
 WorkingDirectory=/home/yellowtent/box
 Restart=always
 ExecStart=/home/yellowtent/box/box.js
-Environment="HOME=/home/yellowtent" "USER=yellowtent" "DEBUG=box:*,connect-lastmile,-box:ldap" "BOX_ENV=cloudron" "NODE_ENV=production"
+; we run commands like df which will parse properly only with correct locale
+Environment="HOME=/home/yellowtent" "USER=yellowtent" "DEBUG=box:*,connect-lastmile,-box:ldap" "BOX_ENV=cloudron" "NODE_ENV=production" "LC_ALL=C"
 ; kill apptask processes as well
 KillMode=control-group
 ; Do not kill this process on OOM. Children inherit this score. Do not set it to -1000 so that MemoryMax can keep working

setup/start/systemd/cloudron-firewall.service

@@ -5,6 +5,7 @@ PartOf=docker.service
 
 [Service]
 Type=oneshot
+Environment="BOX_ENV=cloudron"
 ExecStart="/home/yellowtent/box/setup/start/cloudron-firewall.sh"
 RemainAfterExit=yes
 

src/dns/namecheap.js

@@ -16,10 +16,10 @@ var assert = require('assert'),
     debug = require('debug')('box:dns/namecheap'),
     dns = require('../native-dns.js'),
     domains = require('../domains.js'),
+    querystring = require('querystring'),
     safe = require('safetydance'),
     superagent = require('superagent'),
     sysinfo = require('../sysinfo.js'),
-    util = require('util'),
     waitForDns = require('./waitfordns.js'),
     xml2js = require('xml2js');
 
@@ -50,11 +50,9 @@ function getQuery(dnsConfig, callback) {
     });
 }
 
-function getInternal(dnsConfig, zoneName, subdomain, type, callback) {
+function getZone(dnsConfig, zoneName, callback) {
     assert.strictEqual(typeof dnsConfig, 'object');
     assert.strictEqual(typeof zoneName, 'string');
-    assert.strictEqual(typeof subdomain, 'string');
-    assert.strictEqual(typeof type, 'string');
     assert.strictEqual(typeof callback, 'function');
 
     getQuery(dnsConfig, function (error, query) {
@@ -89,7 +87,7 @@ function getInternal(dnsConfig, zoneName, subdomain, type, callback) {
     });
 }
 
-function setInternal(dnsConfig, zoneName, hosts, callback) {
+function setZone(dnsConfig, zoneName, hosts, callback) {
     assert.strictEqual(typeof dnsConfig, 'object');
     assert.strictEqual(typeof zoneName, 'string');
     assert(Array.isArray(hosts));
@@ -116,7 +114,10 @@ function setInternal(dnsConfig, zoneName, hosts, callback) {
             }
         });
 
-        superagent.post(ENDPOINT).query(query).end(function (error, result) {
+        // namecheap recommends sending as POSTDATA with > 10 records
+        const qs = querystring.stringify(query);
+
+        superagent.post(ENDPOINT).set('Content-Type', 'application/x-www-form-urlencoded').send(qs).end(function (error, result) {
             if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, error));
 
             var parser = new xml2js.Parser();
@@ -144,7 +145,7 @@ function upsert(domainObject, subdomain, type, values, callback) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert(util.isArray(values));
+    assert(Array.isArray(values));
     assert.strictEqual(typeof callback, 'function');
 
     const dnsConfig = domainObject.config;
@@ -154,17 +155,17 @@ function upsert(domainObject, subdomain, type, values, callback) {
 
     debug('upsert: %s for zone %s of type %s with values %j', subdomain, zoneName, type, values);
 
-    getInternal(dnsConfig, zoneName, subdomain, type, function (error, result) {
+    getZone(dnsConfig, zoneName, function (error, result) {
         if (error) return callback(error);
 
         // Array to keep track of records that need to be inserted
         let toInsert = [];
 
-        for (var i = 0; i < values.length; i++) {
+        for (let i = 0; i < values.length; i++) {
             let curValue = values[i];
             let wasUpdate = false;
 
-            for (var j = 0; j < result.length; j++) {
+            for (let j = 0; j < result.length; j++) {
                 let curHost = result[j];
 
                 if (curHost.Type === type && curHost.Name === subdomain) {
@@ -198,9 +199,9 @@ function upsert(domainObject, subdomain, type, values, callback) {
             }
         }
 
-        let toUpsert = result.concat(toInsert);
+        const hosts = result.concat(toInsert);
 
-        setInternal(dnsConfig, zoneName, toUpsert, callback);
+        setZone(dnsConfig, zoneName, hosts, callback);
     });
 }
 
@@ -215,16 +216,16 @@ function get(domainObject, subdomain, type, callback) {
 
     subdomain = domains.getName(domainObject, subdomain, type) || '@';
 
-    getInternal(dnsConfig, zoneName, subdomain, type, function (error, result) {
+    getZone(dnsConfig, zoneName, function (error, result) {
         if (error) return callback(error);
 
         // We need to filter hosts to ones with this subdomain and type
-        let actualHosts = result.filter((host) => host.Type === type && host.Name === subdomain);
+        const actualHosts = result.filter((host) => host.Type === type && host.Name === subdomain);
 
         // We only return the value string
-        var tmp = actualHosts.map(function (record) { return record.Address; });
+        const tmp = actualHosts.map(function (record) { return record.Address; });
 
-        debug('get: %j', tmp);
+        debug(`get: subdomain: ${subdomain} type:${type} value:${JSON.stringify(tmp)}`);
 
         return callback(null, tmp);
     });
@@ -234,7 +235,7 @@ function del(domainObject, subdomain, type, values, callback) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert(util.isArray(values));
+    assert(Array.isArray(values));
     assert.strictEqual(typeof callback, 'function');
 
     const dnsConfig = domainObject.config;
@@ -244,29 +245,19 @@ function del(domainObject, subdomain, type, values, callback) {
 
     debug('del: %s for zone %s of type %s with values %j', subdomain, zoneName, type, values);
 
-    getInternal(dnsConfig, zoneName, subdomain, type, function (error, result) {
+    getZone(dnsConfig, zoneName, function (error, result) {
         if (error) return callback(error);
 
         if (result.length === 0) return callback();
+        const originalLength = result.length;
 
-        let removed = false;
-
-        for (var i = 0; i < values.length; i++) {
+        for (let i = 0; i < values.length; i++) {
             let curValue = values[i];
 
-            for (var j = 0; j < result.length; j++) {
-                let curHost = result[i];
-
-                if (curHost.Type === type && curHost.Name === subdomain && curHost.Address === curValue) {
-                    removed = true;
-
-                    result.splice(i, 1); // Remove element from result array
-                }
-            }
+            result = result.filter(curHost => curHost.Type !== type || curHost.Name !== subdomain || curHost.Address !== curValue);
         }
 
-        // Only set hosts if we actually removed a host
-        if (removed) return setInternal(dnsConfig, zoneName, result, callback);
+        if (result.length !== originalLength) return setZone(dnsConfig, zoneName, result, callback);
 
         callback();
     });

src/infra_version.js

@@ -15,13 +15,13 @@ exports = module.exports = {
     // a major version bump in the db containers will trigger the restore logic that uses the db dumps
     // docker inspect --format='{{index .RepoDigests 0}}' $IMAGE to get the sha256
     'images': {
-        'turn': { repo: 'cloudron/turn', tag: 'cloudron/turn:1.3.0@sha256:386fb755fc41edd7086f7bcb230f7f28078936f9ae4ead6d97c741df1cc194ae' },
+        'turn': { repo: 'cloudron/turn', tag: 'cloudron/turn:1.3.1@sha256:759cafab7625ff538418a1f2ed5558b1d5bff08c576bba577d865d6d02b49091' },
         'mysql': { repo: 'cloudron/mysql', tag: 'cloudron/mysql:3.0.4@sha256:4d688c746f27b195d98f35a7d24ec01f3f754e0ca61e9de0b0bc9793553880f1' },
         'postgresql': { repo: 'cloudron/postgresql', tag: 'cloudron/postgresql:4.0.2@sha256:424081fd38ebd35f3606c64f8f99138570e5f4d5066f12cfb4142447d249d3e7' },
         'mongodb': { repo: 'cloudron/mongodb', tag: 'cloudron/mongodb:4.0.1@sha256:ad20a9a5dcb2ab132374a7c8d44b89af0ec37651cf889e570f7625b02ee85fdf' },
         'redis': { repo: 'cloudron/redis', tag: 'cloudron/redis:3.0.2@sha256:caaa1f7f4055ae8990d8ec65bd100567496df7e4ed5eb427867f3717a8dcbf92' },
         'mail': { repo: 'cloudron/mail', tag: 'cloudron/mail:3.2.3@sha256:fdc4aa6d2c85aeafe65eaa4243aada0cc2e57b94f6eaee02c9b1a8fb89b01dd7' },
-        'graphite': { repo: 'cloudron/graphite', tag: 'cloudron/graphite:3.0.0@sha256:b00b64b8df4032985d7a1ddd548a2713b6d7d88a54ebe9b7d324cece2bd6829e' },
+        'graphite': { repo: 'cloudron/graphite', tag: 'cloudron/graphite:3.0.1@sha256:bed9f6b5d06fe2c5289e895e806cfa5b74ad62993d705be55d4554a67d128029' },
         'sftp': { repo: 'cloudron/sftp', tag: 'cloudron/sftp:3.2.0@sha256:61e8247ded1e07cf882ca478dab180960357c614472e80b938f1f690a46788c2' }
     }
 };

src/network.js

@@ -21,7 +21,7 @@ function getBlocklist(callback) {
     assert.strictEqual(typeof callback, 'function');
 
     const data = safe.fs.readFileSync(paths.FIREWALL_BLOCKLIST_FILE, 'utf8');
-    callback(null, data);
+    callback(null, data || '');
 }
 
 function setBlocklist(blocklist, auditSource, callback) {
@@ -39,8 +39,8 @@ function setBlocklist(blocklist, auditSource, callback) {
         if (rangeOrIP.indexOf('/') === -1) {
             if (auditSource.ip === rangeOrIP) return callback(new BoxError(BoxError.BAD_FIELD, `${rangeOrIP} includes client IP. Cannot block yourself`));
         } else {
-            const parsedRange = ipaddr.parseCIDR(rangeOrIP);
-            if (parsedIp.match(parsedRange)) return callback(new BoxError(BoxError.BAD_FIELD, `${rangeOrIP} includes client IP. Cannot block yourself`));
+            const parsedRange = ipaddr.parseCIDR(rangeOrIP); // returns [addr, range]
+            if (parsedRange[0].kind() === parsedIp.kind() && parsedIp.match(parsedRange)) return callback(new BoxError(BoxError.BAD_FIELD, `${rangeOrIP} includes client IP. Cannot block yourself`));
         }
     }
 
@@ -51,6 +51,6 @@ function setBlocklist(blocklist, auditSource, callback) {
     shell.sudo('setBlocklist', [ SET_BLOCKLIST_CMD ], {}, function (error) {
         if (error) return callback(new BoxError(BoxError.IPTABLES_ERROR, `Error setting blocklist: ${error.message}`));
 
-        callback();
+        callback(null);
     });
 }

src/services.js

@@ -1821,7 +1821,12 @@ function startGraphite(existingInfra, serviceConfig, callback) {
             shell.sudo('removeGraphiteDir', [ RMADDONDIR_CMD, 'graphite' ], {}, done);
         },
         shell.exec.bind(null, 'startGraphite', cmd)
-    ], callback);
+    ], function (error) {
+        // restart collectd to get the disk stats after graphite starts. currently, there is no way to do graphite health check
+        if (!error) setTimeout(() => shell.sudo('restartcollectd', [ RESTART_SERVICE_CMD, 'collectd' ], {}, NOOP_CALLBACK), 60000);
+
+        callback(error);
+    });
 }
 
 function setupProxyAuth(app, options, callback) {
@@ -1863,6 +1868,10 @@ function startRedis(existingInfra, callback) {
             const redisName = 'redis-' + app.id;
 
             async.series([
+                (done) => {
+                    if (!upgrading) return done();
+                    backupRedis(app, {}, done);
+                },
                 shell.exec.bind(null, 'stopRedis', `docker stop ${redisName} || true`), // redis will backup as part of signal handling
                 shell.exec.bind(null, 'removeRedis', `docker rm -f ${redisName} || true`),
                 setupRedis.bind(null, app, app.manifest.addons.redis) // starts the container
@@ -2152,10 +2161,7 @@ function restartGraphite(callback) {
 
     docker.restartContainer('graphite', callback);
 
-    setTimeout(function () {
-        // wait for graphite to startup and then restart collectd
-        shell.sudo('restartcollectd', [ RESTART_SERVICE_CMD, 'collectd' ], {}, NOOP_CALLBACK);
-    }, 10000);
+    setTimeout(() => shell.sudo('restartcollectd', [ RESTART_SERVICE_CMD, 'collectd' ], {}, NOOP_CALLBACK), 60000);
 }
 
 function teardownOauth(app, options, callback) {

src/taskworker.js

@@ -62,7 +62,10 @@ async.series([
     database.initialize,
     settings.initCache
 ], function (error) {
-    if (error) return process.exit(50);
+    if (error) {
+        console.error(error);
+        return process.exit(50);
+    }
 
     const debug = require('debug')('box:taskworker'); // require this here so that logging handler is already setup
     const NOOP_CALLBACK = function (error) { if (error) debug(error); };
@@ -75,11 +78,14 @@ async.series([
     debug(`Starting task ${taskId}. Logs are at ${logFile}`);
 
     tasks.get(taskId, function (error, task) {
-        if (error) return process.exit(50);
+        if (error) {
+            debug(error);
+            return process.exit(50);
+        }
 
         tasks.update(taskId, { percent: 2, error: null }, function (error) {
             if (error) {
-                console.error(error);
+                debug(error);
                 return process.exit(50);
             }
 

src/test/network-test.js

@@ -0,0 +1,196 @@
+/* global it:false */
+/* global describe:false */
+/* global before:false */
+
+'use strict';
+
+var network = require('../network.js'),
+    fs = require('fs'),
+    path = require('path'),
+    paths = require('../paths.js'),
+    BoxError = require('../boxerror.js'),
+    expect = require('expect.js');
+
+describe('Network', function () {
+    describe('Blocklist', function () {
+
+        before(function () {
+            fs.mkdirSync(path.dirname(paths.FIREWALL_BLOCKLIST_FILE));
+        });
+
+        it('can get empty blocklist', function (done) {
+            network.getBlocklist(function (error, result) {
+                expect(error).to.equal(null);
+                expect(result).to.equal('');
+
+                done();
+            });
+        });
+
+        it('can set empty blocklist', function (done) {
+            network.setBlocklist('', { ip: '127.0.0.1' }, function (error) {
+                expect(error).to.equal(null);
+
+                network.getBlocklist(function (error, result) {
+                    expect(error).to.equal(null);
+                    expect(result).to.equal('\n');
+
+                    done();
+                });
+            });
+        });
+
+        it('can set single IPv4 in blocklist', function (done) {
+            network.setBlocklist('192.168.178.1', { ip: '127.0.0.1' }, function (error) {
+                expect(error).to.equal(null);
+
+                network.getBlocklist(function (error, result) {
+                    expect(error).to.equal(null);
+                    expect(result).to.equal('192.168.178.1\n');
+
+                    done();
+                });
+            });
+        });
+
+        it('can set single IPv6 in blocklist', function (done) {
+            network.setBlocklist('2a02:8106:2f:bb00:7afc:5703:ee71:3ef8', { ip: '127.0.0.1' }, function (error) {
+                expect(error).to.equal(null);
+
+                network.getBlocklist(function (error, result) {
+                    expect(error).to.equal(null);
+                    expect(result).to.equal('2a02:8106:2f:bb00:7afc:5703:ee71:3ef8\n');
+
+                    done();
+                });
+            });
+        });
+
+        it('can set mixed IPs with comment in blocklist', function (done) {
+            network.setBlocklist('2a02:8106:2f:bb00:7afc:5703:ee71:3ef8\n# some comment\n192.168.178.1', { ip: '127.0.0.1' }, function (error) {
+                expect(error).to.equal(null);
+
+                network.getBlocklist(function (error, result) {
+                    expect(error).to.equal(null);
+                    expect(result).to.equal('2a02:8106:2f:bb00:7afc:5703:ee71:3ef8\n# some comment\n192.168.178.1\n');
+
+                    done();
+                });
+            });
+        });
+
+        it('can set single IPv4 range in blocklist', function (done) {
+            network.setBlocklist('192.168.178.1/24', { ip: '127.0.0.1' }, function (error) {
+                expect(error).to.equal(null);
+
+                network.getBlocklist(function (error, result) {
+                    expect(error).to.equal(null);
+                    expect(result).to.equal('192.168.178.1/24\n');
+
+                    done();
+                });
+            });
+        });
+
+        it('can set single IPv6 range in blocklist', function (done) {
+            network.setBlocklist('2001:db8::', { ip: '127.0.0.1' }, function (error) {
+                expect(error).to.equal(null);
+
+                network.getBlocklist(function (error, result) {
+                    expect(error).to.equal(null);
+                    expect(result).to.equal('2001:db8::\n');
+
+                    done();
+                });
+            });
+        });
+
+        it('cannot set IPv4 in blocklist if source is same', function (done) {
+            network.setBlocklist('127.0.0.1', { ip: '127.0.0.1' }, function (error) {
+                expect(error).to.be.a(BoxError);
+                expect(error.reason).to.equal(BoxError.BAD_FIELD);
+
+                done();
+            });
+        });
+
+        it('cannot set IPv6 in blocklist if source is same', function (done) {
+            network.setBlocklist('2001:db8:1234::1', { ip: '2001:db8:1234::1' }, function (error) {
+                expect(error).to.be.a(BoxError);
+                expect(error.reason).to.equal(BoxError.BAD_FIELD);
+
+                done();
+            });
+        });
+
+        it('cannot set IPv4 range in blocklist if source is same', function (done) {
+            network.setBlocklist('127.0.0.1/32', { ip: '127.0.0.1' }, function (error) {
+                expect(error).to.be.a(BoxError);
+                expect(error.reason).to.equal(BoxError.BAD_FIELD);
+
+                done();
+            });
+        });
+
+        it('cannot set IPv6 range in blocklist if source is same', function (done) {
+            network.setBlocklist('2001:db8:1234:::', { ip: '2001:db8:1234::1' }, function (error) {
+                expect(error).to.be.a(BoxError);
+                expect(error.reason).to.equal(BoxError.BAD_FIELD);
+
+                done();
+            });
+        });
+
+        it('can set IPv4 in blocklist if source is IPv6', function (done) {
+            network.setBlocklist('192.168.178.1', { ip: '2001:db8:1234::1' }, function (error) {
+                expect(error).to.equal(null);
+
+                network.getBlocklist(function (error, result) {
+                    expect(error).to.equal(null);
+                    expect(result).to.equal('192.168.178.1\n');
+
+                    done();
+                });
+            });
+        });
+
+        it('can set IPv6 in blocklist if source is IPv4', function (done) {
+            network.setBlocklist('2001:db8:1234::1', { ip: '127.0.0.1' }, function (error) {
+                expect(error).to.equal(null);
+
+                network.getBlocklist(function (error, result) {
+                    expect(error).to.equal(null);
+                    expect(result).to.equal('2001:db8:1234::1\n');
+
+                    done();
+                });
+            });
+        });
+
+        it('can set IPv4 range in blocklist if source is IPv6', function (done) {
+            network.setBlocklist('192.168.178.1/32', { ip: '2001:db8:1234::1' }, function (error) {
+                expect(error).to.equal(null);
+
+                network.getBlocklist(function (error, result) {
+                    expect(error).to.equal(null);
+                    expect(result).to.equal('192.168.178.1/32\n');
+
+                    done();
+                });
+            });
+        });
+
+        it('can set IPv6 range in blocklist if source is IPv4', function (done) {
+            network.setBlocklist('2001:db8:1234::', { ip: '127.0.0.1' }, function (error) {
+                expect(error).to.equal(null);
+
+                network.getBlocklist(function (error, result) {
+                    expect(error).to.equal(null);
+                    expect(result).to.equal('2001:db8:1234::\n');
+
+                    done();
+                });
+            });
+        });
+    });
+});