collectd: cache du values and send it every Interval (20)

collectd plugin ordering matters. the write_graphite plugin establishes
a TCP connection but there is a race between that and the df/du values that
get reported. du is especially problematic since we report this only every 12 hours.

so, instead we cache the values and report it every 20 seconds. on the carbon side,
it will just retain every 12 hours (since that is the whisper retention period).

there is also FlushInterval which I am not 100% sure has any effect. by default, the
write_graphite plugin waits for 1428 bytes to be accumulated. (https://manpages.debian.org/unstable/collectd-core/collectd.conf.5.en.html)

https://github.com/collectd/collectd/issues/2672
https://github.com/collectd/collectd/pull/1044

I found this syntax hidden deep inside https://www.cisco.com/c/en/us/td/docs/net_mgmt/virtual_topology_system/2_6_3/user_guide/Cisco_VTS_2_6_3_User_Guide/Cisco_VTS_2_6_1_User_Guide_chapter_01111.pdf

(cherry picked from commit c1ee3dcbd4)

CHANGES

@@ -2238,3 +2238,11 @@
 * sftp: fix rebuild condition
 * Fix display of user management/dashboard visiblity for email apps
 * graphite: disable tagdb and reduce log noise
+
+[6.2.6]
+* Fix issue where collectd is restarted too quickly before graphite
+
+[6.2.7]
+* collectd: restart on graphite upgrade
+
+* 

setup/start/collectd/collectd.conf

@@ -164,7 +164,9 @@ LoadPlugin swap
 #LoadPlugin vmem
 #LoadPlugin vserver
 #LoadPlugin wireless
-LoadPlugin write_graphite
+<LoadPlugin write_graphite>
+   FlushInterval 20
+</LoadPlugin>
 #LoadPlugin write_http
 #LoadPlugin write_riemann
 

setup/start/collectd/du.py

@@ -6,19 +6,26 @@ PATHS = [] # { name, dir, exclude }
 # there is a pattern in carbon/storage-schemas.conf which stores values every 12h for a year
 INTERVAL = 60 * 60 * 12 # twice a day. change values in docker-graphite if you change this
 
+# we used to pass the INTERVAL as a parameter to register_read. however, collectd write_graphite
+# takes a bit to load (tcp connection) and drops the du data. this then means that we have to wait
+# for INTERVAL secs for du data. instead, we just cache the value for INTERVAL instead
+CACHE = dict()
+CACHE_TIME = 0
+
 def du(pathinfo):
     # -B1 makes du print block sizes and not apparent sizes (to match df which also uses block sizes)
-    cmd = 'timeout 1800 du -DsB1 "{}"'.format(pathinfo['dir'])
+    dirname = pathinfo['dir']
+    cmd = 'timeout 1800 du -DsB1 "{}"'.format(dirname)
     if pathinfo['exclude'] != '':
         cmd += ' --exclude "{}"'.format(pathinfo['exclude'])
 
     collectd.info('computing size with command: %s' % cmd);
     try:
         size = subprocess.check_output(cmd, shell=True).split()[0].decode('utf-8')
-        collectd.info('\tsize of %s is %s (time: %i)' % (pathinfo['dir'], size, int(time.time())))
+        collectd.info('\tsize of %s is %s (time: %i)' % (dirname, size, int(time.time())))
         return size
     except Exception as e:
-        collectd.info('\terror getting the size of %s: %s' % (pathinfo['dir'], str(e)))
+        collectd.info('\terror getting the size of %s: %s' % (dirname, str(e)))
         return 0
 
 def parseSize(size):
@@ -64,19 +71,35 @@ def init():
     collectd.info('custom du plugin initialized with %s %s' % (PATHS, sys.version))
 
 def read():
+    global CACHE, CACHE_TIME
+
+    # read from cache if < 12 hours
+    read_cache = (time.time() - CACHE_TIME) < INTERVAL
+
+    if not read_cache:
+        CACHE_TIME = time.time()
+
     for pathinfo in PATHS:
-        size = du(pathinfo)
+        dirname = pathinfo['dir']
+        if read_cache and dirname in CACHE:
+            size = CACHE[dirname]
+        else:
+            size = du(pathinfo)
+            CACHE[dirname] = size
 
         # type comes from https://github.com/collectd/collectd/blob/master/src/types.db
         val = collectd.Values(type='capacity', plugin='du', plugin_instance=pathinfo['name'])
         val.dispatch(values=[size], type_instance='usage')
 
-    size = dockerSize()
+    if read_cache and 'docker' in CACHE:
+        size = CACHE['docker']
+    else:
+        size = dockerSize()
+        CACHE['docker'] = size
+
     val = collectd.Values(type='capacity', plugin='du', plugin_instance='docker')
     val.dispatch(values=[size], type_instance='usage')
 
-
-
 collectd.register_init(init)
 collectd.register_config(configure)
-collectd.register_read(read, INTERVAL)
+collectd.register_read(read)

src/services.js

@@ -1821,7 +1821,12 @@ function startGraphite(existingInfra, serviceConfig, callback) {
             shell.sudo('removeGraphiteDir', [ RMADDONDIR_CMD, 'graphite' ], {}, done);
         },
         shell.exec.bind(null, 'startGraphite', cmd)
-    ], callback);
+    ], function (error) {
+        // restart collectd to get the disk stats after graphite starts. currently, there is no way to do graphite health check
+        if (!error) setTimeout(() => shell.sudo('restartcollectd', [ RESTART_SERVICE_CMD, 'collectd' ], {}, NOOP_CALLBACK), 60000);
+
+        callback(error);
+    });
 }
 
 function setupProxyAuth(app, options, callback) {
@@ -1863,6 +1868,10 @@ function startRedis(existingInfra, callback) {
             const redisName = 'redis-' + app.id;
 
             async.series([
+                (done) => {
+                    if (!upgrading) return done();
+                    backupRedis(app, {}, done);
+                },
                 shell.exec.bind(null, 'stopRedis', `docker stop ${redisName} || true`), // redis will backup as part of signal handling
                 shell.exec.bind(null, 'removeRedis', `docker rm -f ${redisName} || true`),
                 setupRedis.bind(null, app, app.manifest.addons.redis) // starts the container
@@ -2152,10 +2161,7 @@ function restartGraphite(callback) {
 
     docker.restartContainer('graphite', callback);
 
-    setTimeout(function () {
-        // wait for graphite to startup and then restart collectd. graphite migrations can be quite slow!
-        shell.sudo('restartcollectd', [ RESTART_SERVICE_CMD, 'collectd' ], {}, NOOP_CALLBACK);
-    }, 60000);
+    setTimeout(() => shell.sudo('restartcollectd', [ RESTART_SERVICE_CMD, 'collectd' ], {}, NOOP_CALLBACK), 60000);
 }
 
 function teardownOauth(app, options, callback) {