convert cert to pem

this is not used anywhere since we want to wait for rate limits to be
fixed.

The current limits are :

    Rate limit on registrations per IP is currently 10 per 3 hours
    Rate limit on certificates per Domain is currently 5 per 7 days

The domains are counted based on https://publicsuffix.org/list/ (not TLD). Like appspot.com, herokuapp.com while not a TLD, it a public suffix. This list allows browser authors to limit how cookies can be manipulated by the subdomain of those domains. like app1.appspot.com cannot go and change things of app2.appspot.com.

This means
a) we cannot use LE for cloudron.me, cloudron.us (or we have to get on that list)

b) even for custom domains we get only 5 certs every 7 days. And one of them is taken for my.xx domain.

https://community.letsencrypt.org/t/public-beta-rate-limits/4772/38

setup/start.sh

@@ -48,6 +48,7 @@ mkdir -p "${DATA_DIR}/mongodb"
 mkdir -p "${DATA_DIR}/snapshots"
 mkdir -p "${DATA_DIR}/addons"
 mkdir -p "${DATA_DIR}/collectd/collectd.conf.d"
+mkdir -p "${DATA_DIR}/acme"
 
 # bookkeep the version as part of data
 echo "{ \"version\": \"${arg_version}\", \"boxVersionsUrl\": \"${arg_box_versions_url}\" }" > "${DATA_DIR}/box/version"
@@ -125,7 +126,7 @@ else
 fi
 
 set_progress "33" "Changing ownership"
-chown "${USER}:${USER}" -R "${DATA_DIR}/box" "${DATA_DIR}/nginx" "${DATA_DIR}/collectd" "${DATA_DIR}/addons"
+chown "${USER}:${USER}" -R "${DATA_DIR}/box" "${DATA_DIR}/nginx" "${DATA_DIR}/collectd" "${DATA_DIR}/addons" "${DATA_DIR}/acme"
 chown "${USER}:${USER}" "${DATA_DIR}"
 
 set_progress "40" "Setting up infra"

setup/start/nginx/nginx.ejs

@@ -38,6 +38,11 @@ http {
             deny all;
         }
 
+        # acme challenges
+        location /.well-known/acme-challenge/ {
+            alias /home/yellowtent/data/acme/;
+        }
+
         location / {
             # redirect everything to HTTPS
             return 301 https://$host$request_uri;

src/acme.js

@@ -0,0 +1,334 @@
+/* jslint node:true */
+
+'use strict';
+
+var assert = require('assert'),
+    async = require('async'),
+    config = require('./config.js'),
+    crypto = require('crypto'),
+    debug = require('debug')('acme'),
+    execSync = require('child_process').execSync,
+    fs = require('fs'),
+    path = require('path'),
+    paths = require('./paths.js'),
+    superagent = require('superagent'),
+    urlBase64Encode = require('url-base64-node').escape,
+    ursa = require('ursa'),
+    util = require('util'),
+    _ = require('underscore');
+
+var CA_STAGING = 'https://acme-v01.api.letsencrypt.org',
+    CA_STAGING = 'https://acme-staging.api.letsencrypt.org/',
+    LE_AGREEMENT = 'https://letsencrypt.org/documents/LE-SA-v1.0.1-July-27-2015.pdf';
+
+exports = module.exports = {
+    getCertificate: getCertificate
+};
+
+function AcmeError(reason, errorOrMessage) {
+    assert.strictEqual(typeof reason, 'string');
+    assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined');
+
+    Error.call(this);
+    Error.captureStackTrace(this, this.constructor);
+
+    this.name = this.constructor.name;
+    this.reason = reason;
+    if (typeof errorOrMessage === 'undefined') {
+        this.message = reason;
+    } else if (typeof errorOrMessage === 'string') {
+        this.message = errorOrMessage;
+    } else {
+        this.message = 'Internal error';
+        this.nestedError = errorOrMessage;
+    }
+}
+util.inherits(AcmeError, Error);
+AcmeError.INTERNAL_ERROR = 'Internal Error';
+AcmeError.EXTERNAL_ERROR = 'External Error';
+AcmeError.ALREADY_EXISTS = 'Already Exists';
+AcmeError.NOT_COMPLETED = 'Not Completed';
+AcmeError.FORBIDDEN = 'Forbidden';
+
+// http://jose.readthedocs.org/en/latest/
+// https://www.ietf.org/proceedings/92/slides/slides-92-acme-1.pdf
+// https://community.letsencrypt.org/t/list-of-client-implementations/2103
+
+function getNonce(callback) {
+    superagent.get(CA_STAGING + '/directory', function (error, response) {
+        if (error) return callback(error);
+        if (response.statusCode !== 200) return callback(new Error('Invalid response code when fetching nonce : ' + response.statusCode));
+
+        return callback(null, response.headers['Replay-Nonce'.toLowerCase()]);
+    });
+}
+
+// urlsafe base64 encoding (jose)
+function b64(str) {
+    var buf = util.isBuffer(str) ? str : new Buffer(str);
+   return urlBase64Encode(buf.toString('base64'));
+}
+
+function sendSignedRequest(url, privateKeyPem, payload, callback) {
+    assert.strictEqual(typeof url, 'string');
+    assert(util.isBuffer(privateKeyPem));
+    assert.strictEqual(typeof payload, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var privateKey = ursa.createPrivateKey(privateKeyPem);
+
+    var header = {
+        alg: 'RS256',
+        jwk: {
+            e: b64(privateKey.getExponent()),
+            kty: 'RSA',
+            n: b64(privateKey.getModulus())
+        }
+    };
+ 
+    var payload64 = b64(payload);
+
+    getNonce(function (error, nonce) {
+        if (error) return callback(error);
+
+        debug('Using nonce %s', nonce);
+
+        var protected64 = b64(JSON.stringify(_.extend({ }, header, { nonce: nonce })));
+
+        var signer = ursa.createSigner('sha256');
+        signer.update(protected64 + '.' + payload64, 'utf8');
+        var signature64 = urlBase64Encode(signer.sign(privateKey, 'base64'));
+
+        var data = {
+            header: header,
+            protected: protected64,
+            payload: payload64,
+            signature: signature64
+        };
+
+        superagent.post(url).set('Content-Type', 'application/x-www-form-urlencoded').send(JSON.stringify(data)).buffer().end(function (error, res) {
+            if (error && !error.response) return callback(error); // network errors
+
+            callback(null, res);
+        });
+    });
+}
+
+function registerUser(privateKeyPem, email, callback) {
+    assert(util.isBuffer(privateKeyPem));
+    assert.strictEqual(typeof email, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var payload = {
+        resource: 'new-reg',
+        contact: [ 'mailto:' + email ],
+        agreement: LE_AGREEMENT
+    };
+
+    debug('registerUser: %s', email);
+
+    sendSignedRequest(CA_STAGING + '/acme/new-reg', privateKeyPem, JSON.stringify(payload), function (error, result) {
+        if (error) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, 'Network error when registering user: ' + error.message));
+        if (result.statusCode === 409) return callback(new AcmeError(AcmeError.ALREADY_EXISTS, result.body.detail));
+        if (result.statusCode !== 201) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, util.format('Failed to register user. Expecting 201, got %s %s', result.statusCode, result.text)));
+
+        debug('registerUser: registered user %s', email);
+
+        callback();
+    });
+}
+
+function registerDomain(privateKeyPem, domain, callback) {
+    assert(util.isBuffer(privateKeyPem));
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var payload = {
+        resource: 'new-authz',
+        identifier: {
+            type: 'dns',
+            value: domain
+        }
+    };
+
+    debug('registerDomain: %s', domain);
+
+    sendSignedRequest(CA_STAGING + '/acme/new-authz', privateKeyPem, JSON.stringify(payload), function (error, result) {
+        if (error) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, 'Network error when registering domain: ' + error.message));
+        if (result.statusCode === 403) return callback(new AcmeError(AcmeError.FORBIDDEN, result.body.detail));
+        if (result.statusCode !== 201) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, util.format('Failed to register user. Expecting 201, got %s %s', result.statusCode, result.text)));
+
+        debug('registerDomain: registered %s', domain);
+
+        callback(null, result.body);
+    });
+}
+
+function prepareHttpChallenge(privateKeyPem, challenge, callback) {
+    assert(util.isBuffer(privateKeyPem));
+    assert.strictEqual(typeof challenge, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('prepareHttpChallenge: preparing for challenge %j', challenge);
+
+    var token = challenge.token;
+
+    var privateKey = ursa.createPrivateKey(privateKeyPem);
+
+    var jwk = {
+        e: b64(privateKey.getExponent()),
+        kty: 'RSA',
+        n: b64(privateKey.getModulus())
+    };
+
+    var shasum = crypto.createHash('sha256');
+    shasum.update(JSON.stringify(jwk));
+    var thumbprint = urlBase64Encode(shasum.digest('base64'));
+    var keyAuthorization = token + '.' + thumbprint;
+
+    debug('prepareHttpChallenge: writing %s to %s', keyAuthorization, path.join(paths.ACME_CHALLENGES_DIR, token));
+
+    fs.writeFile(path.join(paths.ACME_CHALLENGES_DIR, token), token + '.' + thumbprint, function (error) {
+        if (error) return callback(new AcmeError(AcmeError.INTERNAL_ERROR, error));
+
+        callback();
+    });
+}
+
+function notifyChallengeReady(privateKeyPem, challenge, callback) {
+    assert(util.isBuffer(privateKeyPem));
+    assert.strictEqual(typeof challenge, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('notifyChallengeReady: %s was met', challenge.uri);
+
+    var keyAuthorization = fs.readFileSync(path.join(paths.ACME_CHALLENGES_DIR, challenge.token), 'utf8');
+
+    var payload = {
+        resource: 'challenge',
+        keyAuthorization: keyAuthorization
+    };
+
+    sendSignedRequest(challenge.uri, privateKeyPem, JSON.stringify(payload), function (error, result) {
+        if (error) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, 'Network error when notifying challenge: ' + error.message));
+        if (result.statusCode !== 202) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, util.format('Failed to notify challenge. Expecting 202, got %s %s', result.statusCode, result.text)));
+
+        callback();
+    });
+}
+
+function waitForChallenge(challenge, callback) {
+    assert.strictEqual(typeof challenge, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('waitingForChallenge: %j', challenge);
+
+    async.retry({ times: 10, interval: 5000 }, function (retryCallback) {
+        debug('waitingForChallenge: getting status');
+
+        superagent.get(challenge.uri).end(function (error, result) {
+            if (error && !error.response) {
+                debug('waitForChallenge: network error getting uri %s', challenge.uri);
+                return retryCallback(new AcmeError(AcmeError.EXTERNAL_ERROR, error.message)); // network error
+            }
+            if (result.statusCode !== 202) {
+                debug('waitForChallenge: invalid response code getting uri %s', result.statusCode);
+                return retryCallback(new AcmeError(AcmeError.EXTERNAL_ERROR, 'Bad response code:' + result.statusCode));
+            }
+
+            debug('waitForChallenge: status is "%s"', result.body.status);
+
+            if (result.body.status === 'pending') return retryCallback(new AcmeError(AcmeError.NOT_COMPLETED));
+            else if (result.body.status === 'valid') return retryCallback();
+            else return retryCallback(new AcmeError(AcmeError.EXTERNAL_ERROR, 'Unexpected status: ' + result.body.status));
+        });
+    }, callback);
+}
+
+// https://community.letsencrypt.org/t/public-beta-rate-limits/4772 for rate limits
+function signCertificate(privateKeyPem, certificateDer, callback) {
+    assert(util.isBuffer(privateKeyPem));
+    assert(util.isBuffer(certificateDer));
+    assert.strictEqual(typeof callback, 'function');
+
+    var payload = {
+        resource: 'new-cert',
+        csr: b64(certificateDer)
+    };
+
+    debug('signCertificate: signing %s', payload.csr);
+
+    sendSignedRequest(CA_STAGING + '/acme/new-cert', privateKeyPem, JSON.stringify(payload), function (error, result) {
+        if (error) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, 'Network error when signing certificate: ' + error.message));
+        if (result.statusCode !== 201) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, util.format('Failed to sign certificate. Expecting 201, got %s %s', result.statusCode, result.text)));
+
+        // TODO: result.body can be empty in which case it has to be polled for from this location
+        debug('signCertificate: certificate is available at ', result.headers['location']);
+
+        callback(null, result.text);
+    });
+}
+
+function acmeFlow(domain, email, privateKeyPem, callback) {
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof email, 'string');
+    assert(util.isBuffer(privateKeyPem));
+    assert.strictEqual(typeof callback, 'function');
+
+    registerUser(privateKeyPem, email, function (error) {
+        if (error && error.reason !== AcmeError.ALREADY_EXISTS) return callback(error);
+
+        registerDomain(privateKeyPem, domain, function (error, result) {
+            if (error) return callback(error);
+
+            debug('getCertificate: challenges: %j', result);
+
+            var httpChallenges = result.challenges.filter(function(x) { return x.type === 'http-01'; });
+            if (httpChallenges.length === 0) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, 'no http challenges'));
+            var challenge = httpChallenges[0];
+
+            prepareHttpChallenge(privateKeyPem, challenge, function (error) {
+                if (error) return callback(error);
+
+                notifyChallengeReady(privateKeyPem, challenge, function (error) {
+                    if (error) return callback(error);
+
+                    waitForChallenge(challenge, function (error) {
+                        if (error) return callback(error);
+
+                        var serverKey = execSync('openssl genrsa 4096');
+                        var certificateDer = execSync(util.format('openssl req -nodes -outform DER -subj /CN=%s', domain), { stdio: [ serverKey, null, null ] });
+
+                        signCertificate(privateKeyPem, certificateDer, function (error, certificateDer) {
+                            if (error) return callback(error);
+                            var certificatePem = execSync('openssl x509 -inform DER -outform PEM', { stdio: [ certificateDer, null, null ] });
+
+                            callback(null, serverKey, certificatePem);
+                        });
+                    });
+                });
+            });
+        });
+    });
+}
+
+function getCertificate(domain, callback) {
+    var email = 'admin@' + config.fqdn();
+    var privateKeyPem;
+
+    if (!fs.existsSync(paths.ACME_ACCOUNT_KEY_FILE)) {
+        debug('getCertificate: generating acme account key on first run');
+        privateKeyPem = execSync('openssl genrsa 4096');
+        fs.writeFileSync(paths.ACME_ACCOUNT_KEY_FILE, privateKeyPem);
+    } else {
+        privateKeyPem = fs.readFileSync(paths.ACME_ACCOUNT_KEY_FILE);
+    }
+
+    acmeFlow(domain, email, privateKeyPem, callback);
+}
+
+getCertificate('foobar.girish.in', function (error, key, cert) {
+    console.dir(error);
+    console.dir(key);
+    console.dir(cert);
+});

src/apps.js

@@ -497,8 +497,6 @@ function getLogs(appId, lines, follow, callback) {
         if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND));
         if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
-        if (app.installationState !== appdb.ISTATE_INSTALLED) return callback(new AppsError(AppsError.BAD_STATE, util.format('App is in %s state.', app.installationState)));
-
         var args = [ '--output=json', '--no-pager', '--lines=' + lines ];
         if (follow) args.push('--follow');
         args = args.concat(appLogFilter(app));

src/apptask.js

@@ -660,7 +660,7 @@ function stopApp(app, callback) {
     docker.stopContainers(app.id, function (error) {
         if (error) return callback(error);
 
-        updateApp(app, { runState: appdb.RSTATE_STOPPED }, callback);
+        updateApp(app, { runState: appdb.RSTATE_STOPPED, health: null }, callback);
     });
 }
 

src/database.js

@@ -126,6 +126,8 @@ function clear(callback) {
 function beginTransaction(callback) {
     assert.strictEqual(typeof callback, 'function');
 
+    if (gConnectionPool === null) return callback(new Error('No database connection pool.'));
+
     gConnectionPool.getConnection(function (error, connection) {
         if (error) return callback(error);
 

src/docker.js

@@ -154,6 +154,9 @@ function createSubcontainer(app, name, cmd, options, callback) {
     }
 
     var memoryLimit = manifest.memoryLimit || 1024 * 1024 * 200; // 200mb by default
+    // for subcontainers, this should ideally be false. but docker does not allow network sharing if the app container is not running
+    // this means cloudron exec does not work
+    var isolatedNetworkNs = true;
 
     addons.getEnvironment(app, function (error, addonEnv) {
         if (error) return callback(new Error('Error getting addon environment : ' + error));
@@ -161,7 +164,8 @@ function createSubcontainer(app, name, cmd, options, callback) {
         var containerOptions = {
             name: name, // used for filtering logs
             // do _not_ set hostname to app fqdn. doing so sets up the dns name to look up the internal docker ip. this makes curl from within container fail
-            Hostname: semver.gte(targetBoxVersion(app.manifest), '0.0.77') ? app.location : config.appFqdn(app.location),
+            // for subcontainers, this should not be set because we already share the network namespace with app container
+            Hostname: isolatedNetworkNs ? (semver.gte(targetBoxVersion(app.manifest), '0.0.77') ? app.location : config.appFqdn(app.location)) : null,
             Tty: isAppContainer,
             Image: app.manifest.dockerImage,
             Cmd: cmd,
@@ -183,13 +187,14 @@ function createSubcontainer(app, name, cmd, options, callback) {
                 PortBindings: isAppContainer ? dockerPortBindings : { },
                 PublishAllPorts: false,
                 ReadonlyRootfs: semver.gte(targetBoxVersion(app.manifest), '0.0.66'), // see also Volumes in startContainer
-                Links: addons.getLinksSync(app, app.manifest.addons),
                 RestartPolicy: {
                     "Name": isAppContainer ? "always" : "no",
                     "MaximumRetryCount": 0
                 },
                 CpuShares: 512, // relative to 1024 for system processes
                 VolumesFrom: isAppContainer ? null : [ app.containerId + ":rw" ],
+                NetworkMode: isolatedNetworkNs ? 'default' : ('container:' + app.containerId), // share network namespace with parent
+                Links: isolatedNetworkNs ? addons.getLinksSync(app, app.manifest.addons) : null, // links is redundant with --net=container
                 SecurityOpt: config.CLOUDRON ? [ "apparmor:docker-cloudron-app" ] : null // profile available only on cloudron
             }
         };

src/mail_templates/app_down.ejs

@@ -5,8 +5,7 @@ Dear Admin,
 The application titled '<%= title %>' that you installed at <%= appFqdn %>
 is not responding.
 
-This is most likely a problem in the application. Please report this issue to
-support@cloudron.io (by forwarding this email).
+This is most likely a problem in the application.
 
 You are receiving this email because you are an Admin of the Cloudron at <%= fqdn %>.
 

src/mailer.js

@@ -284,7 +284,7 @@ function appDied(app) {
 
         var mailOptions = {
             from: config.get('adminEmail'),
-            to: adminEmails.join(', '),
+            to: adminEmails.concat('support@cloudron.io').join(', '),
             subject: util.format('App %s is down', app.location),
             text: render('app_down.ejs', { fqdn: config.fqdn(), title: app.manifest.title, appFqdn: config.appFqdn(app.location), format: 'text' })
         };

src/paths.js

@@ -28,5 +28,8 @@ exports = module.exports = {
     CLOUDRON_AVATAR_FILE: path.join(config.baseDir(), 'data/box/avatar.png'),
     CLOUDRON_DEFAULT_AVATAR_FILE: path.join(__dirname + '/../assets/avatar.png'),
 
-    UPDATE_CHECKER_FILE: path.join(config.baseDir(), 'data/box/updatechecker.json')
+    UPDATE_CHECKER_FILE: path.join(config.baseDir(), 'data/box/updatechecker.json'),
+
+    ACME_CHALLENGES_DIR: path.join(config.baseDir(), 'data/acme'),
+    ACME_ACCOUNT_KEY_FILE: path.join(config.baseDir(), 'data/box/acme.key')
 };