Files
cloudron-box/src/cert/acme2.js
Girish Ramakrishnan 07af3edf51 request has no retry method
i thought it was using superagent

(cherry picked from commit 7df89e66c8)
2021-03-22 14:32:57 -07:00

631 lines
26 KiB
JavaScript

'use strict';
var assert = require('assert'),
async = require('async'),
BoxError = require('../boxerror.js'),
crypto = require('crypto'),
debug = require('debug')('box:cert/acme2'),
domains = require('../domains.js'),
fs = require('fs'),
path = require('path'),
paths = require('../paths.js'),
request = require('request'),
safe = require('safetydance'),
util = require('util'),
_ = require('underscore');
const CA_PROD_DIRECTORY_URL = 'https://acme-v02.api.letsencrypt.org/directory',
CA_STAGING_DIRECTORY_URL = 'https://acme-staging-v02.api.letsencrypt.org/directory';
exports = module.exports = {
getCertificate: getCertificate,
// testing
_name: 'acme',
_getChallengeSubdomain: getChallengeSubdomain
};
// http://jose.readthedocs.org/en/latest/
// https://www.ietf.org/proceedings/92/slides/slides-92-acme-1.pdf
// https://community.letsencrypt.org/t/list-of-client-implementations/2103
function Acme2(options) {
assert.strictEqual(typeof options, 'object');
this.accountKeyPem = null; // Buffer
this.email = options.email;
this.keyId = null;
this.caDirectory = options.prod ? CA_PROD_DIRECTORY_URL : CA_STAGING_DIRECTORY_URL;
this.directory = {};
this.performHttpAuthorization = !!options.performHttpAuthorization;
this.wildcard = !!options.wildcard;
}
// urlsafe base64 encoding (jose)
function urlBase64Encode(string) {
return string.replace(/\+/g, '-').replace(/\//g, '_').replace(/=/g, '');
}
function b64(str) {
var buf = util.isBuffer(str) ? str : Buffer.from(str);
return urlBase64Encode(buf.toString('base64'));
}
function getModulus(pem) {
assert(util.isBuffer(pem));
var stdout = safe.child_process.execSync('openssl rsa -modulus -noout', { input: pem, encoding: 'utf8' });
if (!stdout) return null;
var match = stdout.match(/Modulus=([0-9a-fA-F]+)$/m);
if (!match) return null;
return Buffer.from(match[1], 'hex');
}
Acme2.prototype.sendSignedRequest = function (url, payload, callback) {
assert.strictEqual(typeof url, 'string');
assert.strictEqual(typeof payload, 'string');
assert.strictEqual(typeof callback, 'function');
assert(util.isBuffer(this.accountKeyPem));
const that = this;
let header = {
url: url,
alg: 'RS256'
};
// keyId is null when registering account
if (this.keyId) {
header.kid = this.keyId;
} else {
header.jwk = {
e: b64(Buffer.from([0x01, 0x00, 0x01])), // exponent - 65537
kty: 'RSA',
n: b64(getModulus(this.accountKeyPem))
};
}
var payload64 = b64(payload);
request.get(this.directory.newNonce, { json: true, timeout: 30000 }, function (error, response) {
if (error) return callback(new BoxError(BoxError.NETWORK_ERROR, `Network error sending signed request: ${error.message}`));
if (response.statusCode !== 204) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response code when fetching nonce : ' + response.statusCode));
const nonce = response.headers['Replay-Nonce'.toLowerCase()];
if (!nonce) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'No nonce in response'));
debug('sendSignedRequest: using nonce %s for url %s', nonce, url);
var protected64 = b64(JSON.stringify(_.extend({ }, header, { nonce: nonce })));
var signer = crypto.createSign('RSA-SHA256');
signer.update(protected64 + '.' + payload64, 'utf8');
var signature64 = urlBase64Encode(signer.sign(that.accountKeyPem, 'base64'));
var data = {
protected: protected64,
payload: payload64,
signature: signature64
};
request.post(url, { headers: { 'Content-Type': 'application/jose+json', 'User-Agent': 'acme-cloudron' }, body: JSON.stringify(data), timeout: 30000 }, function (error, response) {
if (error) return callback(new BoxError(BoxError.NETWORK_ERROR, `Network error sending signed request: ${error.message}`)); // network error
// we don't set json: true in request because it ends up mangling the content-type
// we don't set json: true in request because it ends up mangling the content-type
if (response.headers['content-type'] === 'application/json') response.body = safe.JSON.parse(response.body);
callback(null, response);
});
});
};
// https://tools.ietf.org/html/rfc8555#section-6.3
Acme2.prototype.postAsGet = function (url, callback) {
this.sendSignedRequest(url, '', callback);
};
Acme2.prototype.updateContact = function (registrationUri, callback) {
assert.strictEqual(typeof registrationUri, 'string');
assert.strictEqual(typeof callback, 'function');
debug(`updateContact: registrationUri: ${registrationUri} email: ${this.email}`);
// https://github.com/ietf-wg-acme/acme/issues/30
const payload = {
contact: [ 'mailto:' + this.email ]
};
const that = this;
this.sendSignedRequest(registrationUri, JSON.stringify(payload), function (error, result) {
if (error) return callback(error);
if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Failed to update contact. Expecting 200, got ${result.statusCode} ${JSON.stringify(result.body)}`));
debug(`updateContact: contact of user updated to ${that.email}`);
callback();
});
};
Acme2.prototype.registerUser = function (callback) {
assert.strictEqual(typeof callback, 'function');
var payload = {
termsOfServiceAgreed: true
};
debug('registerUser: registering user');
var that = this;
this.sendSignedRequest(this.directory.newAccount, JSON.stringify(payload), function (error, result) {
if (error) return callback(error);
// 200 if already exists. 201 for new accounts
if (result.statusCode !== 200 && result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Failed to register new account. Expecting 200 or 201, got ${result.statusCode} ${JSON.stringify(result.body)}`));
debug(`registerUser: user registered keyid: ${result.headers.location}`);
that.keyId = result.headers.location;
that.updateContact(result.headers.location, callback);
});
};
Acme2.prototype.newOrder = function (domain, callback) {
assert.strictEqual(typeof domain, 'string');
assert.strictEqual(typeof callback, 'function');
var payload = {
identifiers: [{
type: 'dns',
value: domain
}]
};
debug('newOrder: %s', domain);
this.sendSignedRequest(this.directory.newOrder, JSON.stringify(payload), function (error, result) {
if (error) return callback(error);
if (result.statusCode === 403) return callback(new BoxError(BoxError.ACCESS_DENIED, `Forbidden sending new order: ${result.body.detail}`));
if (result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Failed to send new order. Expecting 201, got ${result.statusCode} ${JSON.stringify(result.body)}`));
debug('newOrder: created order %s %j', domain, result.body);
const order = result.body, orderUrl = result.headers.location;
if (!Array.isArray(order.authorizations)) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'invalid authorizations in order'));
if (typeof order.finalize !== 'string') return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'invalid finalize in order'));
if (typeof orderUrl !== 'string') return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'invalid order location in order header'));
callback(null, order, orderUrl);
});
};
Acme2.prototype.waitForOrder = function (orderUrl, callback) {
assert.strictEqual(typeof orderUrl, 'string');
assert.strictEqual(typeof callback, 'function');
debug(`waitForOrder: ${orderUrl}`);
const that = this;
async.retry({ times: 15, interval: 20000 }, function (retryCallback) {
debug('waitForOrder: getting status');
that.postAsGet(orderUrl, function (error, result) {
if (error) {
debug('waitForOrder: network error getting uri %s', orderUrl);
return retryCallback(error);
}
if (result.statusCode !== 200) {
debug('waitForOrder: invalid response code getting uri %s', result.statusCode);
return retryCallback(new BoxError(BoxError.EXTERNAL_ERROR, 'Bad response code:' + result.statusCode));
}
debug('waitForOrder: status is "%s %j', result.body.status, result.body);
if (result.body.status === 'pending' || result.body.status === 'processing') return retryCallback(new BoxError(BoxError.TRY_AGAIN, `Request is in ${result.body.status} state`));
else if (result.body.status === 'valid' && result.body.certificate) return retryCallback(null, result.body.certificate);
else return retryCallback(new BoxError(BoxError.EXTERNAL_ERROR, 'Unexpected status or invalid response: ' + result.body));
});
}, callback);
};
Acme2.prototype.getKeyAuthorization = function (token) {
assert(util.isBuffer(this.accountKeyPem));
let jwk = {
e: b64(Buffer.from([0x01, 0x00, 0x01])), // Exponent - 65537
kty: 'RSA',
n: b64(getModulus(this.accountKeyPem))
};
let shasum = crypto.createHash('sha256');
shasum.update(JSON.stringify(jwk));
let thumbprint = urlBase64Encode(shasum.digest('base64'));
return token + '.' + thumbprint;
};
Acme2.prototype.notifyChallengeReady = function (challenge, callback) {
assert.strictEqual(typeof challenge, 'object'); // { type, status, url, token }
assert.strictEqual(typeof callback, 'function');
debug('notifyChallengeReady: %s was met', challenge.url);
const keyAuthorization = this.getKeyAuthorization(challenge.token);
var payload = {
resource: 'challenge',
keyAuthorization: keyAuthorization
};
this.sendSignedRequest(challenge.url, JSON.stringify(payload), function (error, result) {
if (error) return callback(error);
if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Failed to notify challenge. Expecting 200, got ${result.statusCode} ${JSON.stringify(result.body)}`));
callback();
});
};
Acme2.prototype.waitForChallenge = function (challenge, callback) {
assert.strictEqual(typeof challenge, 'object');
assert.strictEqual(typeof callback, 'function');
debug('waitingForChallenge: %j', challenge);
const that = this;
async.retry({ times: 15, interval: 20000 }, function (retryCallback) {
debug('waitingForChallenge: getting status');
that.postAsGet(challenge.url, function (error, result) {
if (error) {
debug('waitForChallenge: network error getting uri %s', challenge.url);
return retryCallback(error);
}
if (result.statusCode !== 200) {
debug('waitForChallenge: invalid response code getting uri %s', result.statusCode);
return retryCallback(new BoxError(BoxError.EXTERNAL_ERROR, 'Bad response code:' + result.statusCode));
}
debug('waitForChallenge: status is "%s" %j', result.body.status, result.body);
if (result.body.status === 'pending') return retryCallback(new BoxError(BoxError.TRY_AGAIN));
else if (result.body.status === 'valid') return retryCallback();
else return retryCallback(new BoxError(BoxError.EXTERNAL_ERROR, 'Unexpected status: ' + result.body.status));
});
}, function retryFinished(error) {
// async.retry will pass 'undefined' as second arg making it unusable with async.waterfall()
callback(error);
});
};
// https://community.letsencrypt.org/t/public-beta-rate-limits/4772 for rate limits
Acme2.prototype.signCertificate = function (domain, finalizationUrl, csrDer, callback) {
assert.strictEqual(typeof domain, 'string');
assert.strictEqual(typeof finalizationUrl, 'string');
assert(util.isBuffer(csrDer));
assert.strictEqual(typeof callback, 'function');
const payload = {
csr: b64(csrDer)
};
debug('signCertificate: sending sign request');
this.sendSignedRequest(finalizationUrl, JSON.stringify(payload), function (error, result) {
if (error) return callback(error);
// 429 means we reached the cert limit for this domain
if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Failed to sign certificate. Expecting 200, got ${result.statusCode} ${JSON.stringify(result.body)}`));
return callback(null);
});
};
Acme2.prototype.createKeyAndCsr = function (hostname, callback) {
assert.strictEqual(typeof hostname, 'string');
assert.strictEqual(typeof callback, 'function');
var outdir = paths.APP_CERTS_DIR;
const certName = hostname.replace('*.', '_.');
var csrFile = path.join(outdir, `${certName}.csr`);
var privateKeyFile = path.join(outdir, `${certName}.key`);
if (safe.fs.existsSync(privateKeyFile)) {
// in some old releases, csr file was corrupt. so always regenerate it
debug('createKeyAndCsr: reuse the key for renewal at %s', privateKeyFile);
} else {
var key = safe.child_process.execSync('openssl ecparam -genkey -name secp384r1'); // openssl ecparam -list_curves
if (!key) return callback(new BoxError(BoxError.OPENSSL_ERROR, safe.error));
if (!safe.fs.writeFileSync(privateKeyFile, key)) return callback(new BoxError(BoxError.FS_ERROR, safe.error));
debug('createKeyAndCsr: key file saved at %s', privateKeyFile);
}
var csrDer = safe.child_process.execSync(`openssl req -new -key ${privateKeyFile} -outform DER -subj /CN=${hostname}`);
if (!csrDer) return callback(new BoxError(BoxError.OPENSSL_ERROR, safe.error));
if (!safe.fs.writeFileSync(csrFile, csrDer)) return callback(new BoxError(BoxError.FS_ERROR, safe.error)); // bookkeeping
debug('createKeyAndCsr: csr file (DER) saved at %s', csrFile);
callback(null, csrDer);
};
Acme2.prototype.downloadCertificate = function (hostname, certUrl, callback) {
assert.strictEqual(typeof hostname, 'string');
assert.strictEqual(typeof certUrl, 'string');
assert.strictEqual(typeof callback, 'function');
var outdir = paths.APP_CERTS_DIR;
const that = this;
async.retry({ times: 5, interval: 20000 }, function (retryCallback) {
debug('downloadCertificate: downloading certificate');
that.postAsGet(certUrl, function (error, result) {
if (error) return retryCallback(new BoxError(BoxError.NETWORK_ERROR, `Network error when downloading certificate: ${error.message}`));
if (result.statusCode === 202) return retryCallback(new BoxError(BoxError.TRY_AGAIN, 'Retry downloading certificate'));
if (result.statusCode !== 200) return retryCallback(new BoxError(BoxError.EXTERNAL_ERROR, `Failed to get cert. Expecting 200, got ${result.statusCode} ${JSON.stringify(result.body)}`));
const fullChainPem = result.body; // buffer
const certName = hostname.replace('*.', '_.');
var certificateFile = path.join(outdir, `${certName}.cert`);
if (!safe.fs.writeFileSync(certificateFile, fullChainPem)) return retryCallback(new BoxError(BoxError.FS_ERROR, safe.error));
debug('downloadCertificate: cert file for %s saved at %s', hostname, certificateFile);
retryCallback(null);
});
}, callback);
};
Acme2.prototype.prepareHttpChallenge = function (hostname, domain, authorization, callback) {
assert.strictEqual(typeof hostname, 'string');
assert.strictEqual(typeof domain, 'string');
assert.strictEqual(typeof authorization, 'object');
assert.strictEqual(typeof callback, 'function');
debug('acmeFlow: challenges: %j', authorization);
let httpChallenges = authorization.challenges.filter(function(x) { return x.type === 'http-01'; });
if (httpChallenges.length === 0) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'no http challenges'));
let challenge = httpChallenges[0];
debug('prepareHttpChallenge: preparing for challenge %j', challenge);
let keyAuthorization = this.getKeyAuthorization(challenge.token);
debug('prepareHttpChallenge: writing %s to %s', keyAuthorization, path.join(paths.ACME_CHALLENGES_DIR, challenge.token));
fs.writeFile(path.join(paths.ACME_CHALLENGES_DIR, challenge.token), keyAuthorization, function (error) {
if (error) return callback(new BoxError(BoxError.FS_ERROR, error));
callback(null, challenge);
});
};
Acme2.prototype.cleanupHttpChallenge = function (hostname, domain, challenge, callback) {
assert.strictEqual(typeof hostname, 'string');
assert.strictEqual(typeof domain, 'string');
assert.strictEqual(typeof challenge, 'object');
assert.strictEqual(typeof callback, 'function');
debug('cleanupHttpChallenge: unlinking %s', path.join(paths.ACME_CHALLENGES_DIR, challenge.token));
fs.unlink(path.join(paths.ACME_CHALLENGES_DIR, challenge.token), callback);
};
function getChallengeSubdomain(hostname, domain) {
let challengeSubdomain;
if (hostname === domain) {
challengeSubdomain = '_acme-challenge';
} else if (hostname.includes('*')) { // wildcard
let subdomain = hostname.slice(0, -domain.length - 1);
challengeSubdomain = subdomain ? subdomain.replace('*', '_acme-challenge') : '_acme-challenge';
} else {
challengeSubdomain = '_acme-challenge.' + hostname.slice(0, -domain.length - 1);
}
debug(`getChallengeSubdomain: challenge subdomain for hostname ${hostname} at domain ${domain} is ${challengeSubdomain}`);
return challengeSubdomain;
}
Acme2.prototype.prepareDnsChallenge = function (hostname, domain, authorization, callback) {
assert.strictEqual(typeof hostname, 'string');
assert.strictEqual(typeof domain, 'string');
assert.strictEqual(typeof authorization, 'object');
assert.strictEqual(typeof callback, 'function');
debug('acmeFlow: challenges: %j', authorization);
let dnsChallenges = authorization.challenges.filter(function(x) { return x.type === 'dns-01'; });
if (dnsChallenges.length === 0) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'no dns challenges'));
let challenge = dnsChallenges[0];
const keyAuthorization = this.getKeyAuthorization(challenge.token);
let shasum = crypto.createHash('sha256');
shasum.update(keyAuthorization);
const txtValue = urlBase64Encode(shasum.digest('base64'));
let challengeSubdomain = getChallengeSubdomain(hostname, domain);
debug(`prepareDnsChallenge: update ${challengeSubdomain} with ${txtValue}`);
domains.upsertDnsRecords(challengeSubdomain, domain, 'TXT', [ `"${txtValue}"` ], function (error) {
if (error) return callback(error);
domains.waitForDnsRecord(challengeSubdomain, domain, 'TXT', txtValue, { times: 200 }, function (error) {
if (error) return callback(error);
callback(null, challenge);
});
});
};
Acme2.prototype.cleanupDnsChallenge = function (hostname, domain, challenge, callback) {
assert.strictEqual(typeof hostname, 'string');
assert.strictEqual(typeof domain, 'string');
assert.strictEqual(typeof challenge, 'object');
assert.strictEqual(typeof callback, 'function');
const keyAuthorization = this.getKeyAuthorization(challenge.token);
let shasum = crypto.createHash('sha256');
shasum.update(keyAuthorization);
const txtValue = urlBase64Encode(shasum.digest('base64'));
let challengeSubdomain = getChallengeSubdomain(hostname, domain);
debug(`cleanupDnsChallenge: remove ${challengeSubdomain} with ${txtValue}`);
domains.removeDnsRecords(challengeSubdomain, domain, 'TXT', [ `"${txtValue}"` ], function (error) {
if (error) return callback(error);
callback(null);
});
};
Acme2.prototype.prepareChallenge = function (hostname, domain, authorizationUrl, callback) {
assert.strictEqual(typeof hostname, 'string');
assert.strictEqual(typeof domain, 'string');
assert.strictEqual(typeof authorizationUrl, 'string');
assert.strictEqual(typeof callback, 'function');
debug(`prepareChallenge: http: ${this.performHttpAuthorization}`);
const that = this;
this.postAsGet(authorizationUrl, function (error, response) {
if (error) return callback(error);
if (response.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response code getting authorization : ' + response.statusCode));
const authorization = response.body;
if (that.performHttpAuthorization) {
that.prepareHttpChallenge(hostname, domain, authorization, callback);
} else {
that.prepareDnsChallenge(hostname, domain, authorization, callback);
}
});
};
Acme2.prototype.cleanupChallenge = function (hostname, domain, challenge, callback) {
assert.strictEqual(typeof hostname, 'string');
assert.strictEqual(typeof domain, 'string');
assert.strictEqual(typeof challenge, 'object');
assert.strictEqual(typeof callback, 'function');
debug(`cleanupChallenge: http: ${this.performHttpAuthorization}`);
if (this.performHttpAuthorization) {
this.cleanupHttpChallenge(hostname, domain, challenge, callback);
} else {
this.cleanupDnsChallenge(hostname, domain, challenge, callback);
}
};
Acme2.prototype.acmeFlow = function (hostname, domain, callback) {
assert.strictEqual(typeof hostname, 'string');
assert.strictEqual(typeof domain, 'string');
assert.strictEqual(typeof callback, 'function');
if (!fs.existsSync(paths.ACME_ACCOUNT_KEY_FILE)) {
debug('getCertificate: generating acme account key on first run');
this.accountKeyPem = safe.child_process.execSync('openssl genrsa 4096');
if (!this.accountKeyPem) return callback(new BoxError(BoxError.OPENSSL_ERROR, safe.error));
safe.fs.writeFileSync(paths.ACME_ACCOUNT_KEY_FILE, this.accountKeyPem);
} else {
debug('getCertificate: using existing acme account key');
this.accountKeyPem = fs.readFileSync(paths.ACME_ACCOUNT_KEY_FILE);
}
var that = this;
this.registerUser(function (error) {
if (error) return callback(error);
that.newOrder(hostname, function (error, order, orderUrl) {
if (error) return callback(error);
async.eachSeries(order.authorizations, function (authorizationUrl, iteratorCallback) {
debug(`acmeFlow: authorizing ${authorizationUrl}`);
that.prepareChallenge(hostname, domain, authorizationUrl, function (error, challenge) {
if (error) return iteratorCallback(error);
async.waterfall([
that.notifyChallengeReady.bind(that, challenge),
that.waitForChallenge.bind(that, challenge),
that.createKeyAndCsr.bind(that, hostname),
that.signCertificate.bind(that, hostname, order.finalize),
that.waitForOrder.bind(that, orderUrl),
that.downloadCertificate.bind(that, hostname)
], function (error) {
that.cleanupChallenge(hostname, domain, challenge, function (cleanupError) {
if (cleanupError) debug('acmeFlow: ignoring error when cleaning up challenge:', cleanupError);
iteratorCallback(error);
});
});
});
}, callback);
});
});
};
Acme2.prototype.getDirectory = function (callback) {
const that = this;
async.retry({ times: 3, interval: 20000 }, function (retryCallback) {
request.get(that.caDirectory, { json: true, timeout: 30000 }, function (error, response) {
if (error) return retryCallback(new BoxError(BoxError.NETWORK_ERROR, `Network error getting directory: ${error.message}`));
if (response.statusCode !== 200) return retryCallback(new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response code when fetching directory : ' + response.statusCode));
if (typeof response.body.newNonce !== 'string' ||
typeof response.body.newOrder !== 'string' ||
typeof response.body.newAccount !== 'string') return retryCallback(new BoxError(BoxError.EXTERNAL_ERROR, `Invalid response body : ${response.body}`));
that.directory = response.body;
retryCallback(null);
});
}, callback);
};
Acme2.prototype.getCertificate = function (vhost, domain, callback) {
assert.strictEqual(typeof vhost, 'string');
assert.strictEqual(typeof domain, 'string');
assert.strictEqual(typeof callback, 'function');
debug(`getCertificate: start acme flow for ${vhost} from ${this.caDirectory}`);
if (vhost !== domain && this.wildcard) { // bare domain is not part of wildcard SAN
vhost = domains.makeWildcard(vhost);
debug(`getCertificate: will get wildcard cert for ${vhost}`);
}
const that = this;
this.getDirectory(function (error) {
if (error) return callback(error);
that.acmeFlow(vhost, domain, function (error) {
if (error) return callback(error);
var outdir = paths.APP_CERTS_DIR;
const certName = vhost.replace('*.', '_.');
callback(null, path.join(outdir, `${certName}.cert`), path.join(outdir, `${certName}.key`));
});
});
};
function getCertificate(vhost, domain, options, callback) {
assert.strictEqual(typeof vhost, 'string'); // this can also be a wildcard domain (for alias domains)
assert.strictEqual(typeof domain, 'string');
assert.strictEqual(typeof options, 'object');
assert.strictEqual(typeof callback, 'function');
let attempt = 1;
async.retry({ times: 3, interval: 0 }, function (retryCallback) {
debug(`getCertificate: attempt ${attempt++}`);
let acme = new Acme2(options || { });
acme.getCertificate(vhost, domain, retryCallback);
}, callback);
}