jjkiers/cloudron-box
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

replace debug() with our custom logger

Browse Source 
mostly we want trace() and log(). trace() can be enabled whenever
we want by flipping a flag and restarting box
This commit is contained in:
Girish Ramakrishnan
2026-03-12 22:55:28 +05:30
parent d57554a48c
commit 01d0c738bc
104 changed files with 1187 additions and 1174 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
box.js
View File

@@ -10,9 +10,9 @@ import proxyAuth from './src/proxyauth.js';
import safe from 'safetydance'; import safe from 'safetydance';
import server from './src/server.js'; import server from './src/server.js';
import directoryServer from './src/directoryserver.js'; import directoryServer from './src/directoryserver.js';
import debugModule from 'debug'; import logger from './src/logger.js';
const debug = debugModule('box:box'); const { log, trace } = logger('box');
let logFd; let logFd;
@@ -59,13 +59,13 @@ const [error] = await safe(startServers());
if (error) exitSync({ error, code: 1, message: 'Error starting servers' }); if (error) exitSync({ error, code: 1, message: 'Error starting servers' });
process.on('SIGHUP', async function () { process.on('SIGHUP', async function () {
debug('Received SIGHUP. Re-reading configs.'); log('Received SIGHUP. Re-reading configs.');
const conf = await directoryServer.getConfig(); const conf = await directoryServer.getConfig();
if (conf.enabled) await directoryServer.checkCertificate(); if (conf.enabled) await directoryServer.checkCertificate();
}); });
process.on('SIGINT', async function () { process.on('SIGINT', async function () {
debug('Received SIGINT. Shutting down.'); log('Received SIGINT. Shutting down.');
await proxyAuth.stop(); await proxyAuth.stop();
await server.stop(); await server.stop();
@@ -74,13 +74,13 @@ process.on('SIGINT', async function () {
await oidcServer.stop(); await oidcServer.stop();
setTimeout(() => { setTimeout(() => {
debug('Shutdown complete'); log('Shutdown complete');
process.exit(); process.exit();
}, 2000); // need to wait for the task processes to die }, 2000); // need to wait for the task processes to die
}); });
process.on('SIGTERM', async function () { process.on('SIGTERM', async function () {
debug('Received SIGTERM. Shutting down.'); log('Received SIGTERM. Shutting down.');
await proxyAuth.stop(); await proxyAuth.stop();
await server.stop(); await server.stop();
@@ -89,7 +89,7 @@ process.on('SIGTERM', async function () {
await oidcServer.stop(); await oidcServer.stop();
setTimeout(() => { setTimeout(() => {
debug('Shutdown complete'); log('Shutdown complete');
process.exit(); process.exit();
}, 2000); // need to wait for the task processes to die }, 2000); // need to wait for the task processes to die
}); });

1
package-lock.json generated
View File

@@ -29,7 +29,6 @@
"cron": "^4.4.0", "cron": "^4.4.0",
"db-migrate": "^0.11.14", "db-migrate": "^0.11.14",
"db-migrate-mysql": "^3.0.0", "db-migrate-mysql": "^3.0.0",
"debug": "^4.4.3",
"dockerode": "^4.0.9", "dockerode": "^4.0.9",
"domrobot-client": "^3.3.0", "domrobot-client": "^3.3.0",
"ejs": "^4.0.1", "ejs": "^4.0.1",

1
package.json
View File

@@ -34,7 +34,6 @@
"cron": "^4.4.0", "cron": "^4.4.0",
"db-migrate": "^0.11.14", "db-migrate": "^0.11.14",
"db-migrate-mysql": "^3.0.0", "db-migrate-mysql": "^3.0.0",
"debug": "^4.4.3",
"dockerode": "^4.0.9", "dockerode": "^4.0.9",
"domrobot-client": "^3.3.0", "domrobot-client": "^3.3.0",
"ejs": "^4.0.1", "ejs": "^4.0.1",

3
setup/start/systemd/box.service
View File

@@ -16,8 +16,7 @@ Restart=always
ExecStart=/home/yellowtent/box/box.js ExecStart=/home/yellowtent/box/box.js
ExecReload=/bin/kill -HUP $MAINPID ExecReload=/bin/kill -HUP $MAINPID
; we run commands like df which will parse properly only with correct locale ; we run commands like df which will parse properly only with correct locale
; add "oidc-provider:*" to DEBUG for OpenID debugging Environment="HOME=/home/yellowtent" "USER=yellowtent" "BOX_ENV=cloudron" "NODE_ENV=production" "LC_ALL=C"
Environment="HOME=/home/yellowtent" "USER=yellowtent" "DEBUG=box:*,-box:ldapserver,-box:directoryserver,-box:oidcserver" "BOX_ENV=cloudron" "NODE_ENV=production" "LC_ALL=C"
; this sends the main process SIGTERM and then if anything lingers to the control-group . this is also the case if the main process crashes. ; this sends the main process SIGTERM and then if anything lingers to the control-group . this is also the case if the main process crashes.
; the box code handles SIGTERM and cleanups the tasks ; the box code handles SIGTERM and cleanups the tasks
KillMode=mixed KillMode=mixed

2
setup/start/systemd/cloudron-syslog.service
View File

@@ -5,7 +5,7 @@ After=network.target
[Service] [Service]
ExecStart=/home/yellowtent/box/syslog.js ExecStart=/home/yellowtent/box/syslog.js
WorkingDirectory=/home/yellowtent/box WorkingDirectory=/home/yellowtent/box
Environment="NODE_ENV=production" "DEBUG=syslog:*" "BOX_ENV=cloudron" Environment="NODE_ENV=production" "BOX_ENV=cloudron"
Restart=always Restart=always
User=yellowtent User=yellowtent
Group=yellowtent Group=yellowtent

84
src/acme2.js
View File

@@ -2,7 +2,7 @@ import assert from 'node:assert';
import blobs from './blobs.js'; import blobs from './blobs.js';
import BoxError from './boxerror.js'; import BoxError from './boxerror.js';
import crypto from 'node:crypto'; import crypto from 'node:crypto';
import debugModule from 'debug'; import logger from './logger.js';
import dns from './dns.js'; import dns from './dns.js';
import openssl from './openssl.js'; import openssl from './openssl.js';
import path from 'node:path'; import path from 'node:path';
@@ -12,7 +12,7 @@ import safe from 'safetydance';
import superagent from '@cloudron/superagent'; import superagent from '@cloudron/superagent';
import users from './users.js'; import users from './users.js';
const debug = debugModule('box:cert/acme2'); const { log, trace } = logger('cert/acme2');
const CA_PROD_DIRECTORY_URL = 'https://acme-v02.api.letsencrypt.org/directory', const CA_PROD_DIRECTORY_URL = 'https://acme-v02.api.letsencrypt.org/directory',
CA_STAGING_DIRECTORY_URL = 'https://acme-staging-v02.api.letsencrypt.org/directory'; CA_STAGING_DIRECTORY_URL = 'https://acme-staging-v02.api.letsencrypt.org/directory';
@@ -53,7 +53,7 @@ function Acme2(fqdn, domainObject, email, key, options) {
this.profile = options.profile || ''; // https://letsencrypt.org/docs/profiles/ . is validated against the directory this.profile = options.profile || ''; // https://letsencrypt.org/docs/profiles/ . is validated against the directory
debug(`Acme2: will get cert for fqdn: ${this.fqdn} cn: ${this.cn} certName: ${this.certName} wildcard: ${this.wildcard} http: ${this.forceHttpAuthorization}`); log(`Acme2: will get cert for fqdn: ${this.fqdn} cn: ${this.cn} certName: ${this.certName} wildcard: ${this.wildcard} http: ${this.forceHttpAuthorization}`);
} }
// urlsafe base64 encoding (jose) // urlsafe base64 encoding (jose)
@@ -140,7 +140,7 @@ Acme2.prototype.sendSignedRequest = async function (url, payload) {
const nonce = response.headers['Replay-Nonce'.toLowerCase()]; const nonce = response.headers['Replay-Nonce'.toLowerCase()];
if (!nonce) throw new BoxError(BoxError.ACME_ERROR, 'No nonce in response'); if (!nonce) throw new BoxError(BoxError.ACME_ERROR, 'No nonce in response');
debug(`sendSignedRequest: using nonce ${nonce} for url ${url}`); log(`sendSignedRequest: using nonce ${nonce} for url ${url}`);
const protected64 = b64(JSON.stringify(Object.assign({}, header, { nonce: nonce }))); const protected64 = b64(JSON.stringify(Object.assign({}, header, { nonce: nonce })));
@@ -168,7 +168,7 @@ Acme2.prototype.postAsGet = async function (url) {
Acme2.prototype.updateContact = async function (registrationUri) { Acme2.prototype.updateContact = async function (registrationUri) {
assert.strictEqual(typeof registrationUri, 'string'); assert.strictEqual(typeof registrationUri, 'string');
debug(`updateContact: registrationUri: ${registrationUri} email: ${this.email}`); log(`updateContact: registrationUri: ${registrationUri} email: ${this.email}`);
// https://github.com/ietf-wg-acme/acme/issues/30 // https://github.com/ietf-wg-acme/acme/issues/30
const payload = { const payload = {
@@ -178,7 +178,7 @@ Acme2.prototype.updateContact = async function (registrationUri) {
const result = await this.sendSignedRequest(registrationUri, JSON.stringify(payload)); const result = await this.sendSignedRequest(registrationUri, JSON.stringify(payload));
if (result.status !== 200) throw new BoxError(BoxError.ACME_ERROR, `Failed to update contact. Expecting 200, got ${result.status} ${result.text}`); if (result.status !== 200) throw new BoxError(BoxError.ACME_ERROR, `Failed to update contact. Expecting 200, got ${result.status} ${result.text}`);
debug(`updateContact: contact of user updated to ${this.email}`); log(`updateContact: contact of user updated to ${this.email}`);
}; };
Acme2.prototype.ensureAccount = async function () { Acme2.prototype.ensureAccount = async function () {
@@ -186,18 +186,18 @@ Acme2.prototype.ensureAccount = async function () {
termsOfServiceAgreed: true termsOfServiceAgreed: true
}; };
debug('ensureAccount: registering user'); log('ensureAccount: registering user');
this.accountKey = await blobs.getString(blobs.ACME_ACCOUNT_KEY); this.accountKey = await blobs.getString(blobs.ACME_ACCOUNT_KEY);
if (!this.accountKey) { if (!this.accountKey) {
debug('ensureAccount: generating new account keys'); log('ensureAccount: generating new account keys');
this.accountKey = await openssl.generateKey('rsa4096'); this.accountKey = await openssl.generateKey('rsa4096');
await blobs.setString(blobs.ACME_ACCOUNT_KEY, this.accountKey); await blobs.setString(blobs.ACME_ACCOUNT_KEY, this.accountKey);
} }
let result = await this.sendSignedRequest(this.directory.newAccount, JSON.stringify(payload)); let result = await this.sendSignedRequest(this.directory.newAccount, JSON.stringify(payload));
if (result.status === 403 && result.body.type === 'urn:ietf:params:acme:error:unauthorized') { if (result.status === 403 && result.body.type === 'urn:ietf:params:acme:error:unauthorized') {
debug(`ensureAccount: key was revoked. ${result.status} ${result.text}. generating new account key`); log(`ensureAccount: key was revoked. ${result.status} ${result.text}. generating new account key`);
this.accountKey = await openssl.generateKey('rsa4096'); this.accountKey = await openssl.generateKey('rsa4096');
await blobs.setString(blobs.ACME_ACCOUNT_KEY, this.accountKey); await blobs.setString(blobs.ACME_ACCOUNT_KEY, this.accountKey);
result = await this.sendSignedRequest(this.directory.newAccount, JSON.stringify(payload)); result = await this.sendSignedRequest(this.directory.newAccount, JSON.stringify(payload));
@@ -206,7 +206,7 @@ Acme2.prototype.ensureAccount = async function () {
// 200 if already exists. 201 for new accounts // 200 if already exists. 201 for new accounts
if (result.status !== 200 && result.status !== 201) throw new BoxError(BoxError.ACME_ERROR, `Failed to register new account. Expecting 200 or 201, got ${result.status} ${result.text}`); if (result.status !== 200 && result.status !== 201) throw new BoxError(BoxError.ACME_ERROR, `Failed to register new account. Expecting 200 or 201, got ${result.status} ${result.text}`);
debug(`ensureAccount: user registered keyid: ${result.headers.location}`); log(`ensureAccount: user registered keyid: ${result.headers.location}`);
this.accountKeyId = result.headers.location; this.accountKeyId = result.headers.location;
@@ -223,14 +223,14 @@ Acme2.prototype.newOrder = async function () {
}); });
}); });
debug(`newOrder: ${JSON.stringify(this.altNames)}`); log(`newOrder: ${JSON.stringify(this.altNames)}`);
const result = await this.sendSignedRequest(this.directory.newOrder, JSON.stringify(payload)); const result = await this.sendSignedRequest(this.directory.newOrder, JSON.stringify(payload));
if (result.status === 403) throw new BoxError(BoxError.ACCESS_DENIED, `Forbidden sending new order: ${result.body.detail}`); if (result.status === 403) throw new BoxError(BoxError.ACCESS_DENIED, `Forbidden sending new order: ${result.body.detail}`);
if (result.status !== 201) throw new BoxError(BoxError.ACME_ERROR, `Failed to send new order. Expecting 201, got ${result.status} ${result.text}`); if (result.status !== 201) throw new BoxError(BoxError.ACME_ERROR, `Failed to send new order. Expecting 201, got ${result.status} ${result.text}`);
const order = result.body, orderUrl = result.headers.location; const order = result.body, orderUrl = result.headers.location;
debug(`newOrder: created order ${this.cn} order: ${result.text} orderUrl: ${orderUrl}`); log(`newOrder: created order ${this.cn} order: ${result.text} orderUrl: ${orderUrl}`);
if (!Array.isArray(order.authorizations)) throw new BoxError(BoxError.ACME_ERROR, 'invalid authorizations in order'); if (!Array.isArray(order.authorizations)) throw new BoxError(BoxError.ACME_ERROR, 'invalid authorizations in order');
if (typeof order.finalize !== 'string') throw new BoxError(BoxError.ACME_ERROR, 'invalid finalize in order'); if (typeof order.finalize !== 'string') throw new BoxError(BoxError.ACME_ERROR, 'invalid finalize in order');
@@ -242,18 +242,18 @@ Acme2.prototype.newOrder = async function () {
Acme2.prototype.waitForOrder = async function (orderUrl) { Acme2.prototype.waitForOrder = async function (orderUrl) {
assert.strictEqual(typeof orderUrl, 'string'); assert.strictEqual(typeof orderUrl, 'string');
debug(`waitForOrder: ${orderUrl}`); log(`waitForOrder: ${orderUrl}`);
return await promiseRetry({ times: 15, interval: 20000, debug }, async () => { return await promiseRetry({ times: 15, interval: 20000, debug: log }, async () => {
debug('waitForOrder: getting status'); log('waitForOrder: getting status');
const result = await this.postAsGet(orderUrl); const result = await this.postAsGet(orderUrl);
if (result.status !== 200) { if (result.status !== 200) {
debug(`waitForOrder: invalid response code getting uri ${result.status}`); log(`waitForOrder: invalid response code getting uri ${result.status}`);
throw new BoxError(BoxError.ACME_ERROR, `Bad response when waiting for order. code: ${result.status}`); throw new BoxError(BoxError.ACME_ERROR, `Bad response when waiting for order. code: ${result.status}`);
} }
debug('waitForOrder: status is "%s %j', result.body.status, result.body); log('waitForOrder: status is "%s %j', result.body.status, result.body);
if (result.body.status === 'pending' || result.body.status === 'processing') throw new BoxError(BoxError.ACME_ERROR, `Request is in ${result.body.status} state`); if (result.body.status === 'pending' || result.body.status === 'processing') throw new BoxError(BoxError.ACME_ERROR, `Request is in ${result.body.status} state`);
else if (result.body.status === 'valid' && result.body.certificate) return result.body.certificate; else if (result.body.status === 'valid' && result.body.certificate) return result.body.certificate;
@@ -279,7 +279,7 @@ Acme2.prototype.getKeyAuthorization = async function (token) {
Acme2.prototype.notifyChallengeReady = async function (challenge) { Acme2.prototype.notifyChallengeReady = async function (challenge) {
assert.strictEqual(typeof challenge, 'object'); // { type, status, url, token } assert.strictEqual(typeof challenge, 'object'); // { type, status, url, token }
debug(`notifyChallengeReady: ${challenge.url} was met`); log(`notifyChallengeReady: ${challenge.url} was met`);
const keyAuthorization = await this.getKeyAuthorization(challenge.token); const keyAuthorization = await this.getKeyAuthorization(challenge.token);
@@ -295,18 +295,18 @@ Acme2.prototype.notifyChallengeReady = async function (challenge) {
Acme2.prototype.waitForChallenge = async function (challenge) { Acme2.prototype.waitForChallenge = async function (challenge) {
assert.strictEqual(typeof challenge, 'object'); assert.strictEqual(typeof challenge, 'object');
debug(`waitingForChallenge: ${JSON.stringify(challenge)}`); log(`waitingForChallenge: ${JSON.stringify(challenge)}`);
await promiseRetry({ times: 15, interval: 20000, debug }, async () => { await promiseRetry({ times: 15, interval: 20000, debug: log }, async () => {
debug('waitingForChallenge: getting status'); log('waitingForChallenge: getting status');
const result = await this.postAsGet(challenge.url); const result = await this.postAsGet(challenge.url);
if (result.status !== 200) { if (result.status !== 200) {
debug(`waitForChallenge: invalid response code getting uri ${result.status}`); log(`waitForChallenge: invalid response code getting uri ${result.status}`);
throw new BoxError(BoxError.ACME_ERROR, `Bad response code when waiting for challenge : ${result.status}`); throw new BoxError(BoxError.ACME_ERROR, `Bad response code when waiting for challenge : ${result.status}`);
} }
debug(`waitForChallenge: status is "${result.body.status}" "${result.text}"`); log(`waitForChallenge: status is "${result.body.status}" "${result.text}"`);
if (result.body.status === 'pending') throw new BoxError(BoxError.ACME_ERROR, 'Challenge is in pending state'); if (result.body.status === 'pending') throw new BoxError(BoxError.ACME_ERROR, 'Challenge is in pending state');
else if (result.body.status === 'valid') return; else if (result.body.status === 'valid') return;
@@ -325,7 +325,7 @@ Acme2.prototype.signCertificate = async function (finalizationUrl, csrPem) {
csr: b64(csrDer) csr: b64(csrDer)
}; };
debug(`signCertificate: sending sign request to ${finalizationUrl}`); log(`signCertificate: sending sign request to ${finalizationUrl}`);
const result = await this.sendSignedRequest(finalizationUrl, JSON.stringify(payload)); const result = await this.sendSignedRequest(finalizationUrl, JSON.stringify(payload));
// 429 means we reached the cert limit for this domain // 429 means we reached the cert limit for this domain
@@ -335,8 +335,8 @@ Acme2.prototype.signCertificate = async function (finalizationUrl, csrPem) {
Acme2.prototype.downloadCertificate = async function (certUrl) { Acme2.prototype.downloadCertificate = async function (certUrl) {
assert.strictEqual(typeof certUrl, 'string'); assert.strictEqual(typeof certUrl, 'string');
return await promiseRetry({ times: 5, interval: 20000, debug }, async () => { return await promiseRetry({ times: 5, interval: 20000, debug: log }, async () => {
debug(`downloadCertificate: downloading certificate of ${this.cn}`); log(`downloadCertificate: downloading certificate of ${this.cn}`);
const result = await this.postAsGet(certUrl); const result = await this.postAsGet(certUrl);
if (result.status === 202) throw new BoxError(BoxError.ACME_ERROR, 'Retry downloading certificate'); if (result.status === 202) throw new BoxError(BoxError.ACME_ERROR, 'Retry downloading certificate');
@@ -350,12 +350,12 @@ Acme2.prototype.downloadCertificate = async function (certUrl) {
Acme2.prototype.prepareHttpChallenge = async function (challenge) { Acme2.prototype.prepareHttpChallenge = async function (challenge) {
assert.strictEqual(typeof challenge, 'object'); assert.strictEqual(typeof challenge, 'object');
debug(`prepareHttpChallenge: preparing for challenge ${JSON.stringify(challenge)}`); log(`prepareHttpChallenge: preparing for challenge ${JSON.stringify(challenge)}`);
const keyAuthorization = await this.getKeyAuthorization(challenge.token); const keyAuthorization = await this.getKeyAuthorization(challenge.token);
const challengeFilePath = path.join(paths.ACME_CHALLENGES_DIR, challenge.token); const challengeFilePath = path.join(paths.ACME_CHALLENGES_DIR, challenge.token);
debug(`prepareHttpChallenge: writing ${keyAuthorization} to ${challengeFilePath}`); log(`prepareHttpChallenge: writing ${keyAuthorization} to ${challengeFilePath}`);
if (!safe.fs.writeFileSync(challengeFilePath, keyAuthorization)) throw new BoxError(BoxError.FS_ERROR, `Error writing challenge: ${safe.error.message}`); if (!safe.fs.writeFileSync(challengeFilePath, keyAuthorization)) throw new BoxError(BoxError.FS_ERROR, `Error writing challenge: ${safe.error.message}`);
}; };
@@ -364,7 +364,7 @@ Acme2.prototype.cleanupHttpChallenge = async function (challenge) {
assert.strictEqual(typeof challenge, 'object'); assert.strictEqual(typeof challenge, 'object');
const challengeFilePath = path.join(paths.ACME_CHALLENGES_DIR, challenge.token); const challengeFilePath = path.join(paths.ACME_CHALLENGES_DIR, challenge.token);
debug(`cleanupHttpChallenge: unlinking ${challengeFilePath}`); log(`cleanupHttpChallenge: unlinking ${challengeFilePath}`);
if (!safe.fs.unlinkSync(challengeFilePath)) throw new BoxError(BoxError.FS_ERROR, `Error unlinking challenge: ${safe.error.message}`); if (!safe.fs.unlinkSync(challengeFilePath)) throw new BoxError(BoxError.FS_ERROR, `Error unlinking challenge: ${safe.error.message}`);
}; };
@@ -381,7 +381,7 @@ function getChallengeSubdomain(cn, domain) {
challengeSubdomain = '_acme-challenge.' + cn.slice(0, -domain.length - 1); challengeSubdomain = '_acme-challenge.' + cn.slice(0, -domain.length - 1);
} }
debug(`getChallengeSubdomain: challenge subdomain for cn ${cn} at domain ${domain} is ${challengeSubdomain}`); log(`getChallengeSubdomain: challenge subdomain for cn ${cn} at domain ${domain} is ${challengeSubdomain}`);
return challengeSubdomain; return challengeSubdomain;
} }
@@ -389,7 +389,7 @@ function getChallengeSubdomain(cn, domain) {
Acme2.prototype.prepareDnsChallenge = async function (cn, challenge) { Acme2.prototype.prepareDnsChallenge = async function (cn, challenge) {
assert.strictEqual(typeof challenge, 'object'); assert.strictEqual(typeof challenge, 'object');
debug(`prepareDnsChallenge: preparing for challenge: ${JSON.stringify(challenge)}`); log(`prepareDnsChallenge: preparing for challenge: ${JSON.stringify(challenge)}`);
const keyAuthorization = await this.getKeyAuthorization(challenge.token); const keyAuthorization = await this.getKeyAuthorization(challenge.token);
const shasum = crypto.createHash('sha256'); const shasum = crypto.createHash('sha256');
@@ -398,7 +398,7 @@ Acme2.prototype.prepareDnsChallenge = async function (cn, challenge) {
const txtValue = urlBase64Encode(shasum.digest('base64')); const txtValue = urlBase64Encode(shasum.digest('base64'));
const challengeSubdomain = getChallengeSubdomain(cn, this.domain); const challengeSubdomain = getChallengeSubdomain(cn, this.domain);
debug(`prepareDnsChallenge: update ${challengeSubdomain} with ${txtValue}`); log(`prepareDnsChallenge: update ${challengeSubdomain} with ${txtValue}`);
await dns.upsertDnsRecords(challengeSubdomain, this.domain, 'TXT', [ `"${txtValue}"` ]); await dns.upsertDnsRecords(challengeSubdomain, this.domain, 'TXT', [ `"${txtValue}"` ]);
@@ -416,7 +416,7 @@ Acme2.prototype.cleanupDnsChallenge = async function (cn, challenge) {
const txtValue = urlBase64Encode(shasum.digest('base64')); const txtValue = urlBase64Encode(shasum.digest('base64'));
const challengeSubdomain = getChallengeSubdomain(cn, this.domain); const challengeSubdomain = getChallengeSubdomain(cn, this.domain);
debug(`cleanupDnsChallenge: remove ${challengeSubdomain} with ${txtValue}`); log(`cleanupDnsChallenge: remove ${challengeSubdomain} with ${txtValue}`);
await dns.removeDnsRecords(challengeSubdomain, this.domain, 'TXT', [ `"${txtValue}"` ]); await dns.removeDnsRecords(challengeSubdomain, this.domain, 'TXT', [ `"${txtValue}"` ]);
}; };
@@ -425,7 +425,7 @@ Acme2.prototype.prepareChallenge = async function (cn, authorization) {
assert.strictEqual(typeof cn, 'string'); assert.strictEqual(typeof cn, 'string');
assert.strictEqual(typeof authorization, 'object'); assert.strictEqual(typeof authorization, 'object');
debug(`prepareChallenge: http: ${this.forceHttpAuthorization} cn: ${cn} authorization: ${JSON.stringify(authorization)}`); log(`prepareChallenge: http: ${this.forceHttpAuthorization} cn: ${cn} authorization: ${JSON.stringify(authorization)}`);
// validation is cached by LE for 60 days or so. if a user switches from non-wildcard DNS (http challenge) to programmatic DNS (dns challenge), then // validation is cached by LE for 60 days or so. if a user switches from non-wildcard DNS (http challenge) to programmatic DNS (dns challenge), then
// LE remembers the challenge type and won't give us a dns challenge for 60 days! // LE remembers the challenge type and won't give us a dns challenge for 60 days!
@@ -447,7 +447,7 @@ Acme2.prototype.cleanupChallenge = async function (cn, challenge) {
assert.strictEqual(typeof cn, 'string'); assert.strictEqual(typeof cn, 'string');
assert.strictEqual(typeof challenge, 'object'); assert.strictEqual(typeof challenge, 'object');
debug(`cleanupChallenge: http: ${this.forceHttpAuthorization}`); log(`cleanupChallenge: http: ${this.forceHttpAuthorization}`);
if (this.forceHttpAuthorization) { if (this.forceHttpAuthorization) {
await this.cleanupHttpChallenge(challenge); await this.cleanupHttpChallenge(challenge);
@@ -461,7 +461,7 @@ Acme2.prototype.acmeFlow = async function () {
const { order, orderUrl } = await this.newOrder(); const { order, orderUrl } = await this.newOrder();
for (const authorizationUrl of order.authorizations) { for (const authorizationUrl of order.authorizations) {
debug(`acmeFlow: authorizing ${authorizationUrl}`); log(`acmeFlow: authorizing ${authorizationUrl}`);
const response = await this.postAsGet(authorizationUrl); const response = await this.postAsGet(authorizationUrl);
if (response.status !== 200) throw new BoxError(BoxError.ACME_ERROR, `Invalid response code getting authorization : ${response.status}`); if (response.status !== 200) throw new BoxError(BoxError.ACME_ERROR, `Invalid response code getting authorization : ${response.status}`);
@@ -471,7 +471,7 @@ Acme2.prototype.acmeFlow = async function () {
const challenge = await this.prepareChallenge(cn, authorization); const challenge = await this.prepareChallenge(cn, authorization);
await this.notifyChallengeReady(challenge); await this.notifyChallengeReady(challenge);
await this.waitForChallenge(challenge); await this.waitForChallenge(challenge);
await safe(this.cleanupChallenge(cn, challenge), { debug }); await safe(this.cleanupChallenge(cn, challenge), { debug: log });
} }
const csr = await openssl.createCsr(this.key, this.cn, this.altNames); const csr = await openssl.createCsr(this.key, this.cn, this.altNames);
@@ -485,7 +485,7 @@ Acme2.prototype.acmeFlow = async function () {
}; };
Acme2.prototype.loadDirectory = async function () { Acme2.prototype.loadDirectory = async function () {
await promiseRetry({ times: 3, interval: 20000, debug }, async () => { await promiseRetry({ times: 3, interval: 20000, debug: log }, async () => {
const response = await superagent.get(this.caDirectory).timeout(30000).ok(() => true); const response = await superagent.get(this.caDirectory).timeout(30000).ok(() => true);
if (response.status !== 200) throw new BoxError(BoxError.ACME_ERROR, `Invalid response code when fetching directory : ${response.status}`); if (response.status !== 200) throw new BoxError(BoxError.ACME_ERROR, `Invalid response code when fetching directory : ${response.status}`);
@@ -502,11 +502,11 @@ Acme2.prototype.loadDirectory = async function () {
}; };
Acme2.prototype.getCertificate = async function () { Acme2.prototype.getCertificate = async function () {
debug(`getCertificate: start acme flow for ${this.cn} from ${this.caDirectory}`); log(`getCertificate: start acme flow for ${this.cn} from ${this.caDirectory}`);
await this.loadDirectory(); await this.loadDirectory();
const result = await this.acmeFlow(); // { key, cert, csr, renewalInfo } const result = await this.acmeFlow(); // { key, cert, csr, renewalInfo }
debug(`getCertificate: acme flow completed for ${this.cn}. renewalInfo: ${JSON.stringify(result.renewalInfo)}`); log(`getCertificate: acme flow completed for ${this.cn}. renewalInfo: ${JSON.stringify(result.renewalInfo)}`);
return result; return result;
}; };
@@ -522,8 +522,8 @@ async function getCertificate(fqdn, domainObject, key) {
const owner = await users.getOwner(); const owner = await users.getOwner();
const email = owner?.email || 'webmaster@cloudron.io'; // can error if not activated yet const email = owner?.email || 'webmaster@cloudron.io'; // can error if not activated yet
return await promiseRetry({ times: 3, interval: 0, debug }, async function () { return await promiseRetry({ times: 3, interval: 0, debug: log }, async function () {
debug(`getCertificate: for fqdn ${fqdn} and domain ${domainObject.domain}`); log(`getCertificate: for fqdn ${fqdn} and domain ${domainObject.domain}`);
const acme = new Acme2(fqdn, domainObject, email, key, { /* profile: 'shortlived' */ }); const acme = new Acme2(fqdn, domainObject, email, key, { /* profile: 'shortlived' */ });
return await acme.getCertificate(); return await acme.getCertificate();

20
src/apphealthmonitor.js
View File

@@ -3,13 +3,13 @@ import assert from 'node:assert';
import AuditSource from './auditsource.js'; import AuditSource from './auditsource.js';
import BoxError from './boxerror.js'; import BoxError from './boxerror.js';
import constants from './constants.js'; import constants from './constants.js';
import debugModule from 'debug'; import logger from './logger.js';
import docker from './docker.js'; import docker from './docker.js';
import eventlog from './eventlog.js'; import eventlog from './eventlog.js';
import safe from 'safetydance'; import safe from 'safetydance';
import superagent from '@cloudron/superagent'; import superagent from '@cloudron/superagent';
const debug = debugModule('box:apphealthmonitor'); const { log, trace } = logger('apphealthmonitor');
const UNHEALTHY_THRESHOLD = 20 * 60 * 1000; // 20 minutes const UNHEALTHY_THRESHOLD = 20 * 60 * 1000; // 20 minutes
@@ -32,20 +32,20 @@ async function setHealth(app, health) {
if (health === apps.HEALTH_HEALTHY) { if (health === apps.HEALTH_HEALTHY) {
healthTime = now; healthTime = now;
if (lastHealth && lastHealth !== apps.HEALTH_HEALTHY) { // app starts out with null health if (lastHealth && lastHealth !== apps.HEALTH_HEALTHY) { // app starts out with null health
debug(`setHealth: ${app.id} (${app.fqdn}) switched from ${lastHealth} to healthy`); log(`setHealth: ${app.id} (${app.fqdn}) switched from ${lastHealth} to healthy`);
// do not send mails for dev apps // do not send mails for dev apps
if (!app.debugMode) await eventlog.add(eventlog.ACTION_APP_UP, AuditSource.HEALTH_MONITOR, { app }); if (!app.debugMode) await eventlog.add(eventlog.ACTION_APP_UP, AuditSource.HEALTH_MONITOR, { app });
} }
} else if (Math.abs(now - healthTime) > UNHEALTHY_THRESHOLD) { } else if (Math.abs(now - healthTime) > UNHEALTHY_THRESHOLD) {
if (lastHealth === apps.HEALTH_HEALTHY) { if (lastHealth === apps.HEALTH_HEALTHY) {
debug(`setHealth: marking ${app.id} (${app.fqdn}) as unhealthy since not seen for more than ${UNHEALTHY_THRESHOLD/(60 * 1000)} minutes`); log(`setHealth: marking ${app.id} (${app.fqdn}) as unhealthy since not seen for more than ${UNHEALTHY_THRESHOLD/(60 * 1000)} minutes`);
// do not send mails for dev apps // do not send mails for dev apps
if (!app.debugMode) await eventlog.add(eventlog.ACTION_APP_DOWN, AuditSource.HEALTH_MONITOR, { app }); if (!app.debugMode) await eventlog.add(eventlog.ACTION_APP_DOWN, AuditSource.HEALTH_MONITOR, { app });
} }
} else { } else {
debug(`setHealth: ${app.id} (${app.fqdn}) waiting for ${(UNHEALTHY_THRESHOLD - Math.abs(now - healthTime))/1000} to update health`); log(`setHealth: ${app.id} (${app.fqdn}) waiting for ${(UNHEALTHY_THRESHOLD - Math.abs(now - healthTime))/1000} to update health`);
return; return;
} }
@@ -137,7 +137,7 @@ async function processDockerEvents(options) {
const now = Date.now(); const now = Date.now();
const notifyUser = !info?.app?.debugMode && ((now - gLastOomMailTime) > OOM_EVENT_LIMIT); const notifyUser = !info?.app?.debugMode && ((now - gLastOomMailTime) > OOM_EVENT_LIMIT);
debug(`OOM ${program} notifyUser: ${notifyUser}. lastOomTime: ${gLastOomMailTime} (now: ${now})`); log(`OOM ${program} notifyUser: ${notifyUser}. lastOomTime: ${gLastOomMailTime} (now: ${now})`);
if (notifyUser) { if (notifyUser) {
await eventlog.add(eventlog.ACTION_APP_OOM, AuditSource.HEALTH_MONITOR, { event, containerId, addonName: info?.addonName || null, app: info?.app || null }); await eventlog.add(eventlog.ACTION_APP_OOM, AuditSource.HEALTH_MONITOR, { event, containerId, addonName: info?.addonName || null, app: info?.app || null });
@@ -147,11 +147,11 @@ async function processDockerEvents(options) {
}); });
stream.on('error', function (error) { stream.on('error', function (error) {
debug('Error reading docker events: %o', error); log('Error reading docker events: %o', error);
}); });
stream.on('end', function () { stream.on('end', function () {
// debug('Event stream ended'); // log('Event stream ended');
}); });
// safety hatch if 'until' doesn't work (there are cases where docker is working with a different time) // safety hatch if 'until' doesn't work (there are cases where docker is working with a different time)
@@ -167,12 +167,12 @@ async function processApp(options) {
const results = await Promise.allSettled(healthChecks); // wait for all promises to finish const results = await Promise.allSettled(healthChecks); // wait for all promises to finish
const unfulfilled = results.filter(r => r.status === 'rejected'); const unfulfilled = results.filter(r => r.status === 'rejected');
if (unfulfilled.length) debug(`app health: ${unfulfilled.length} health checks exceptions. e.g. ${unfulfilled[0].reason}`); // this should not happen if (unfulfilled.length) log(`app health: ${unfulfilled.length} health checks exceptions. e.g. ${unfulfilled[0].reason}`); // this should not happen
const stopped = allApps.filter(app => app.runState === apps.RSTATE_STOPPED); const stopped = allApps.filter(app => app.runState === apps.RSTATE_STOPPED);
const running = allApps.filter(function (a) { return a.installationState === apps.ISTATE_INSTALLED && a.runState === apps.RSTATE_RUNNING && a.health === apps.HEALTH_HEALTHY; }); const running = allApps.filter(function (a) { return a.installationState === apps.ISTATE_INSTALLED && a.runState === apps.RSTATE_RUNNING && a.health === apps.HEALTH_HEALTHY; });
debug(`app health: ${running.length} running / ${stopped.length} stopped / ${allApps.length - running.length - stopped.length} unresponsive`); log(`app health: ${running.length} running / ${stopped.length} stopped / ${allApps.length - running.length - stopped.length} unresponsive`);
} }
async function run(intervalSecs) { async function run(intervalSecs) {

20
src/applinks.js
View File

@@ -3,12 +3,12 @@ import apps from './apps.js';
import BoxError from './boxerror.js'; import BoxError from './boxerror.js';
import crypto from 'node:crypto'; import crypto from 'node:crypto';
import database from './database.js'; import database from './database.js';
import debugModule from 'debug'; import logger from './logger.js';
import jsdom from 'jsdom'; import jsdom from 'jsdom';
import safe from 'safetydance'; import safe from 'safetydance';
import superagent from '@cloudron/superagent'; import superagent from '@cloudron/superagent';
const debug = debugModule('box:applinks'); const { log, trace } = logger('applinks');
const APPLINKS_FIELDS= [ 'id', 'accessRestrictionJson', 'creationTime', 'updateTime', 'ts', 'label', 'tagsJson', 'icon', 'upstreamUri' ].join(','); const APPLINKS_FIELDS= [ 'id', 'accessRestrictionJson', 'creationTime', 'updateTime', 'ts', 'label', 'tagsJson', 'icon', 'upstreamUri' ].join(',');
@@ -84,7 +84,7 @@ async function detectMetaInfo(upstreamUri) {
const [upstreamError, upstreamRespose] = await safe(superagent.get(upstreamUri).set('User-Agent', 'Mozilla').timeout(10*1000)); const [upstreamError, upstreamRespose] = await safe(superagent.get(upstreamUri).set('User-Agent', 'Mozilla').timeout(10*1000));
if (upstreamError) { if (upstreamError) {
debug(`detectMetaInfo: error fetching ${upstreamUri}: ${upstreamError.status}`); log(`detectMetaInfo: error fetching ${upstreamUri}: ${upstreamError.status}`);
return null; return null;
} }
@@ -125,21 +125,21 @@ async function detectMetaInfo(upstreamUri) {
if (favicon) { if (favicon) {
favicon = new URL(favicon, upstreamRespose.url).toString(); favicon = new URL(favicon, upstreamRespose.url).toString();
debug(`detectMetaInfo: found icon: ${favicon}`); log(`detectMetaInfo: found icon: ${favicon}`);
const [error, response] = await safe(superagent.get(favicon).timeout(10*1000)); const [error, response] = await safe(superagent.get(favicon).timeout(10*1000));
if (error) debug(`Failed to fetch icon ${favicon}: ${error.message} ${error.status}`); if (error) log(`Failed to fetch icon ${favicon}: ${error.message} ${error.status}`);
else if (response.headers['content-type']?.indexOf('image') !== -1) icon = response.body || response.text; else if (response.headers['content-type']?.indexOf('image') !== -1) icon = response.body || response.text;
else debug(`Failed to fetch icon ${favicon}: status=${response.status}`); else log(`Failed to fetch icon ${favicon}: status=${response.status}`);
} }
if (!favicon) { if (!favicon) {
debug(`Unable to find a suitable icon for ${upstreamUri}, try fallback favicon.ico`); log(`Unable to find a suitable icon for ${upstreamUri}, try fallback favicon.ico`);
const [error, response] = await safe(superagent.get(`${upstreamUri}/favicon.ico`).timeout(10*1000)); const [error, response] = await safe(superagent.get(`${upstreamUri}/favicon.ico`).timeout(10*1000));
if (error) debug(`Failed to fetch icon ${favicon}: ${error.message}`); if (error) log(`Failed to fetch icon ${favicon}: ${error.message}`);
else if (response.headers['content-type']?.indexOf('image') !== -1) icon = response.body || response.text; else if (response.headers['content-type']?.indexOf('image') !== -1) icon = response.body || response.text;
else debug(`Failed to fetch icon ${favicon}: status=${response.status} content type ${response.headers['content-type']}`); else log(`Failed to fetch icon ${favicon}: status=${response.status} content type ${response.headers['content-type']}`);
} }
// detect label // detect label
@@ -153,7 +153,7 @@ async function detectMetaInfo(upstreamUri) {
async function add(applink) { async function add(applink) {
assert.strictEqual(typeof applink, 'object'); assert.strictEqual(typeof applink, 'object');
debug(`add: ${applink.upstreamUri}`); log(`add: ${applink.upstreamUri}`);
let error = validateUpstreamUri(applink.upstreamUri); let error = validateUpstreamUri(applink.upstreamUri);
if (error) throw error; if (error) throw error;

60
src/apps.js
View File

@@ -10,7 +10,7 @@ import crypto from 'node:crypto';
import { CronTime } from 'cron'; import { CronTime } from 'cron';
import dashboard from './dashboard.js'; import dashboard from './dashboard.js';
import database from './database.js'; import database from './database.js';
import debugModule from 'debug'; import logger from './logger.js';
import dns from './dns.js'; import dns from './dns.js';
import docker from './docker.js'; import docker from './docker.js';
import domains from './domains.js'; import domains from './domains.js';
@@ -40,7 +40,7 @@ import util from 'node:util';
import volumes from './volumes.js'; import volumes from './volumes.js';
import _ from './underscore.js'; import _ from './underscore.js';
const debug = debugModule('box:apps'); const { log, trace } = logger('apps');
const shell = shellModule('apps'); const shell = shellModule('apps');
const PORT_TYPE_TCP = 'tcp'; const PORT_TYPE_TCP = 'tcp';
@@ -306,7 +306,7 @@ function getDuplicateErrorDetails(errorMessage, locations, portBindings) {
const match = errorMessage.match(/Duplicate entry '(.*)' for key '(.*)'/); const match = errorMessage.match(/Duplicate entry '(.*)' for key '(.*)'/);
if (!match) { if (!match) {
debug('Unexpected SQL error message.', errorMessage); log('Unexpected SQL error message.', errorMessage);
return new BoxError(BoxError.DATABASE_ERROR, errorMessage); return new BoxError(BoxError.DATABASE_ERROR, errorMessage);
} }
@@ -1058,7 +1058,7 @@ async function onTaskFinished(error, appId, installationState, taskId, auditSour
switch (installationState) { switch (installationState) {
case ISTATE_PENDING_DATA_DIR_MIGRATION: case ISTATE_PENDING_DATA_DIR_MIGRATION:
if (success) await safe(services.rebuildService('sftp', auditSource), { debug }); if (success) await safe(services.rebuildService('sftp', auditSource), { debug: log });
break; break;
case ISTATE_PENDING_UPDATE: { case ISTATE_PENDING_UPDATE: {
const fromManifest = success ? task.args[1].updateConfig.manifest : app.manifest; const fromManifest = success ? task.args[1].updateConfig.manifest : app.manifest;
@@ -1071,8 +1071,8 @@ async function onTaskFinished(error, appId, installationState, taskId, auditSour
} }
// this can race with an new install task but very unlikely // this can race with an new install task but very unlikely
debug(`onTaskFinished: checking to stop unused services. hasPending: ${appTaskManager.hasPendingTasks()}`) log(`onTaskFinished: checking to stop unused services. hasPending: ${appTaskManager.hasPendingTasks()}`)
if (!appTaskManager.hasPendingTasks()) safe(services.stopUnusedServices(), { debug }); if (!appTaskManager.hasPendingTasks()) safe(services.stopUnusedServices(), { debug: log });
} }
async function getCount() { async function getCount() {
@@ -1452,7 +1452,7 @@ async function startExec(app, execId, options) {
// there is a race where resizing too early results in a 404 "no such exec" // there is a race where resizing too early results in a 404 "no such exec"
// https://git.cloudron.io/cloudron/box/issues/549 // https://git.cloudron.io/cloudron/box/issues/549
setTimeout(async function () { setTimeout(async function () {
await safe(docker.resizeExec(execId, { h: options.rows, w: options.columns }, { debug })); await safe(docker.resizeExec(execId, { h: options.rows, w: options.columns }, { debug: log }));
}, 2000); }, 2000);
} }
@@ -1568,7 +1568,7 @@ async function uploadFile(app, sourceFilePath, destFilePath) {
// the built-in bash printf understands "%q" but not /usr/bin/printf. // the built-in bash printf understands "%q" but not /usr/bin/printf.
// ' gets replaced with '\'' . the first closes the quote and last one starts a new one // ' gets replaced with '\'' . the first closes the quote and last one starts a new one
const escapedDestFilePath = await shell.bash(`printf %q '${destFilePath.replace(/'/g, '\'\\\'\'')}'`, { encoding: 'utf8' }); const escapedDestFilePath = await shell.bash(`printf %q '${destFilePath.replace(/'/g, '\'\\\'\'')}'`, { encoding: 'utf8' });
debug(`uploadFile: ${sourceFilePath} -> ${escapedDestFilePath}`); log(`uploadFile: ${sourceFilePath} -> ${escapedDestFilePath}`);
const execId = await createExec(app, { cmd: [ 'bash', '-c', `cat - > ${escapedDestFilePath}` ], tty: false }); const execId = await createExec(app, { cmd: [ 'bash', '-c', `cat - > ${escapedDestFilePath}` ], tty: false });
const destStream = await startExec(app, execId, { tty: false }); const destStream = await startExec(app, execId, { tty: false });
@@ -1702,9 +1702,9 @@ async function scheduleTask(appId, installationState, taskId, auditSource) {
const options = { timeout: 20 * 60 * 60 * 1000 /* 20 hours */, nice: 15, memoryLimit }; const options = { timeout: 20 * 60 * 60 * 1000 /* 20 hours */, nice: 15, memoryLimit };
appTaskManager.scheduleTask(appId, taskId, options, async function (error) { appTaskManager.scheduleTask(appId, taskId, options, async function (error) {
debug(`scheduleTask: task ${taskId} of ${appId} completed. error: %o`, error); log(`scheduleTask: task ${taskId} of ${appId} completed. error: %o`, error);
if (error?.code === tasks.ECRASHED || error?.code === tasks.ESTOPPED) { // if task crashed, update the error if (error?.code === tasks.ECRASHED || error?.code === tasks.ESTOPPED) { // if task crashed, update the error
debug(`Apptask crashed/stopped: ${error.message}`); log(`Apptask crashed/stopped: ${error.message}`);
const appError = { const appError = {
message: error.message, message: error.message,
reason: BoxError.TASK_ERROR, reason: BoxError.TASK_ERROR,
@@ -1713,12 +1713,12 @@ async function scheduleTask(appId, installationState, taskId, auditSource) {
taskId, taskId,
installationState installationState
}; };
await safe(update(appId, { installationState: ISTATE_ERROR, error: appError, taskId: null }), { debug }); await safe(update(appId, { installationState: ISTATE_ERROR, error: appError, taskId: null }), { debug: log });
} else if (!(installationState === ISTATE_PENDING_UNINSTALL && !error)) { // clear out taskId except for successful uninstall } else if (!(installationState === ISTATE_PENDING_UNINSTALL && !error)) { // clear out taskId except for successful uninstall
await safe(update(appId, { taskId: null }), { debug }); await safe(update(appId, { taskId: null }), { debug: log });
} }
await safe(onTaskFinished(error, appId, installationState, taskId, auditSource), { debug }); // ignore error await safe(onTaskFinished(error, appId, installationState, taskId, auditSource), { debug: log }); // ignore error
}); });
} }
@@ -1740,7 +1740,7 @@ async function addTask(appId, installationState, task, auditSource) {
if (updateError && updateError.reason === BoxError.NOT_FOUND) throw new BoxError(BoxError.BAD_STATE, 'Another task is scheduled for this app'); // could be because app went away OR a taskId exists if (updateError && updateError.reason === BoxError.NOT_FOUND) throw new BoxError(BoxError.BAD_STATE, 'Another task is scheduled for this app'); // could be because app went away OR a taskId exists
if (updateError) throw updateError; if (updateError) throw updateError;
if (scheduleNow) await safe(scheduleTask(appId, installationState, taskId, auditSource), { debug }); // ignore error if (scheduleNow) await safe(scheduleTask(appId, installationState, taskId, auditSource), { debug: log }); // ignore error
return taskId; return taskId;
} }
@@ -1882,7 +1882,7 @@ async function install(data, auditSource) {
if (constants.DEMO && (await getCount() >= constants.DEMO_APP_LIMIT)) throw new BoxError(BoxError.BAD_STATE, 'Too many installed apps, please uninstall a few and try again'); if (constants.DEMO && (await getCount() >= constants.DEMO_APP_LIMIT)) throw new BoxError(BoxError.BAD_STATE, 'Too many installed apps, please uninstall a few and try again');
const appId = crypto.randomUUID(); const appId = crypto.randomUUID();
debug(`Installing app ${appId}`); log(`Installing app ${appId}`);
const app = { const app = {
accessRestriction, accessRestriction,
@@ -2566,7 +2566,7 @@ async function exportApp(app, backupSiteId, auditSource) {
if (!canBackupApp(app)) throw new BoxError(BoxError.BAD_STATE, 'App cannot be backed up in this state'); if (!canBackupApp(app)) throw new BoxError(BoxError.BAD_STATE, 'App cannot be backed up in this state');
const taskId = await tasks.add(`${tasks.TASK_APP_BACKUP_PREFIX}${app.id}`, [ appId, backupSiteId, { snapshotOnly: true } ]); const taskId = await tasks.add(`${tasks.TASK_APP_BACKUP_PREFIX}${app.id}`, [ appId, backupSiteId, { snapshotOnly: true } ]);
safe(tasks.startTask(taskId, {}), { debug }); // background safe(tasks.startTask(taskId, {}), { debug: log }); // background
return { taskId }; return { taskId };
} }
@@ -2887,7 +2887,7 @@ async function getBackupDownloadStream(app, backupId) {
const stream = await backupSites.storageApi(backupSite).download(backupSite.config, downloadBackup.remotePath); const stream = await backupSites.storageApi(backupSite).download(backupSite.config, downloadBackup.remotePath);
stream.on('error', function(error) { stream.on('error', function(error) {
debug(`getBackupDownloadStream: read stream error: ${error.message}`); log(`getBackupDownloadStream: read stream error: ${error.message}`);
ps.emit('error', new BoxError(BoxError.EXTERNAL_ERROR, error)); ps.emit('error', new BoxError(BoxError.EXTERNAL_ERROR, error));
}); });
stream.pipe(ps); stream.pipe(ps);
@@ -2926,11 +2926,11 @@ async function restoreApps(apps, options, auditSource) {
requireNullTaskId: false // ignore existing stale taskId requireNullTaskId: false // ignore existing stale taskId
}; };
debug(`restoreApps: marking ${app.fqdn} as ${installationState} using restore config ${JSON.stringify(restoreConfig)}`); log(`restoreApps: marking ${app.fqdn} as ${installationState} using restore config ${JSON.stringify(restoreConfig)}`);
const [addTaskError, taskId] = await safe(addTask(app.id, installationState, task, auditSource)); const [addTaskError, taskId] = await safe(addTask(app.id, installationState, task, auditSource));
if (addTaskError) debug(`restoreApps: error marking ${app.fqdn} for restore: ${JSON.stringify(addTaskError)}`); if (addTaskError) log(`restoreApps: error marking ${app.fqdn} for restore: ${JSON.stringify(addTaskError)}`);
else debug(`restoreApps: marked ${app.id} as ${installationState} with taskId ${taskId}`); else log(`restoreApps: marked ${app.id} as ${installationState} with taskId ${taskId}`);
} }
} }
@@ -2945,7 +2945,7 @@ async function configureApps(apps, options, auditSource) {
const scheduleNow = !!options.scheduleNow; const scheduleNow = !!options.scheduleNow;
for (const app of apps) { for (const app of apps) {
debug(`configureApps: marking ${app.fqdn} for reconfigure (scheduleNow: ${scheduleNow})`); log(`configureApps: marking ${app.fqdn} for reconfigure (scheduleNow: ${scheduleNow})`);
const task = { const task = {
args: {}, args: {},
@@ -2955,8 +2955,8 @@ async function configureApps(apps, options, auditSource) {
}; };
const [addTaskError, taskId] = await safe(addTask(app.id, ISTATE_PENDING_CONFIGURE, task, auditSource)); const [addTaskError, taskId] = await safe(addTask(app.id, ISTATE_PENDING_CONFIGURE, task, auditSource));
if (addTaskError) debug(`configureApps: error marking ${app.fqdn} for configure: ${JSON.stringify(addTaskError)}`); if (addTaskError) log(`configureApps: error marking ${app.fqdn} for configure: ${JSON.stringify(addTaskError)}`);
else debug(`configureApps: marked ${app.id} for re-configure with taskId ${taskId}`); else log(`configureApps: marked ${app.id} for re-configure with taskId ${taskId}`);
} }
} }
@@ -2972,7 +2972,7 @@ async function restartAppsUsingAddons(changedAddons, auditSource) {
apps = apps.filter(app => app.runState !== RSTATE_STOPPED); // don't start stopped apps apps = apps.filter(app => app.runState !== RSTATE_STOPPED); // don't start stopped apps
for (const app of apps) { for (const app of apps) {
debug(`restartAppsUsingAddons: marking ${app.fqdn} for restart`); log(`restartAppsUsingAddons: marking ${app.fqdn} for restart`);
const task = { const task = {
args: {}, args: {},
@@ -2981,27 +2981,27 @@ async function restartAppsUsingAddons(changedAddons, auditSource) {
// stop apps before updating the databases because postgres will "lock" them preventing import // stop apps before updating the databases because postgres will "lock" them preventing import
const [stopError] = await safe(docker.stopContainers(app.id)); const [stopError] = await safe(docker.stopContainers(app.id));
if (stopError) debug(`restartAppsUsingAddons: error stopping ${app.fqdn}`, stopError); if (stopError) log(`restartAppsUsingAddons: error stopping ${app.fqdn}`, stopError);
const [addTaskError, taskId] = await safe(addTask(app.id, ISTATE_PENDING_RESTART, task, auditSource)); const [addTaskError, taskId] = await safe(addTask(app.id, ISTATE_PENDING_RESTART, task, auditSource));
if (addTaskError) debug(`restartAppsUsingAddons: error marking ${app.fqdn} for restart: ${JSON.stringify(addTaskError)}`); if (addTaskError) log(`restartAppsUsingAddons: error marking ${app.fqdn} for restart: ${JSON.stringify(addTaskError)}`);
else debug(`restartAppsUsingAddons: marked ${app.id} for restart with taskId ${taskId}`); else log(`restartAppsUsingAddons: marked ${app.id} for restart with taskId ${taskId}`);
} }
} }
async function schedulePendingTasks(auditSource) { async function schedulePendingTasks(auditSource) {
assert.strictEqual(typeof auditSource, 'object'); assert.strictEqual(typeof auditSource, 'object');
debug('schedulePendingTasks: scheduling app tasks'); log('schedulePendingTasks: scheduling app tasks');
const result = await list(); const result = await list();
for (const app of result) { for (const app of result) {
if (!app.taskId) continue; // if not in any pending state, do nothing if (!app.taskId) continue; // if not in any pending state, do nothing
debug(`schedulePendingTasks: schedule task for ${app.fqdn} ${app.id}: state=${app.installationState},taskId=${app.taskId}`); log(`schedulePendingTasks: schedule task for ${app.fqdn} ${app.id}: state=${app.installationState},taskId=${app.taskId}`);
await safe(scheduleTask(app.id, app.installationState, app.taskId, auditSource), { debug }); // ignore error await safe(scheduleTask(app.id, app.installationState, app.taskId, auditSource), { debug: log }); // ignore error
} }
} }

24
src/appstore.js
View File

@@ -4,7 +4,7 @@ import backupSites from './backupsites.js';
import BoxError from './boxerror.js'; import BoxError from './boxerror.js';
import constants from './constants.js'; import constants from './constants.js';
import dashboard from './dashboard.js'; import dashboard from './dashboard.js';
import debugModule from 'debug'; import logger from './logger.js';
import domains from './domains.js'; import domains from './domains.js';
import dockerRegistries from './dockerregistries.js'; import dockerRegistries from './dockerregistries.js';
import directoryServer from './directoryserver.js'; import directoryServer from './directoryserver.js';
@@ -23,7 +23,7 @@ import system from './system.js';
import users from './users.js'; import users from './users.js';
import volumes from './volumes.js'; import volumes from './volumes.js';
const debug = debugModule('box:appstore'); const { log, trace } = logger('appstore');
// These are the default options and will be adjusted once a subscription state is obtained // These are the default options and will be adjusted once a subscription state is obtained
// Keep in sync with appstore/routes/cloudrons.js // Keep in sync with appstore/routes/cloudrons.js
@@ -129,7 +129,7 @@ async function getSubscription() {
if (!token) throw new BoxError(BoxError.LICENSE_ERROR, 'Missing token'); if (!token) throw new BoxError(BoxError.LICENSE_ERROR, 'Missing token');
const [stateError, state] = await safe(getState()); const [stateError, state] = await safe(getState());
if (stateError) debug('getSubscription: error getting current state', stateError); if (stateError) log('getSubscription: error getting current state', stateError);
const [error, response] = await safe(superagent.post(`${await getApiServerOrigin()}/api/v1/subscription3`) const [error, response] = await safe(superagent.post(`${await getApiServerOrigin()}/api/v1/subscription3`)
.query({ accessToken: token }) .query({ accessToken: token })
@@ -157,8 +157,8 @@ async function getSubscription() {
// cron hook // cron hook
async function checkSubscription() { async function checkSubscription() {
const [error, result] = await safe(getSubscription()); const [error, result] = await safe(getSubscription());
if (error) debug('checkSubscription error:', error); if (error) log('checkSubscription error:', error);
else debug(`checkSubscription: Cloudron ${result.cloudronId} is on the ${result.plan.name} plan.`); else log(`checkSubscription: Cloudron ${result.cloudronId} is on the ${result.plan.name} plan.`);
} }
function isFreePlan(subscription) { function isFreePlan(subscription) {
@@ -236,7 +236,7 @@ async function getAppUpdate(app, options) {
// do some sanity checks // do some sanity checks
if (!safe.query(updateInfo, 'manifest.version') || semver.gt(curAppVersion, safe.query(updateInfo, 'manifest.version'))) { if (!safe.query(updateInfo, 'manifest.version') || semver.gt(curAppVersion, safe.query(updateInfo, 'manifest.version'))) {
debug('Skipping malformed update of app %s version: %s. got %j', app.id, curAppVersion, updateInfo); log('Skipping malformed update of app %s version: %s. got %j', app.id, curAppVersion, updateInfo);
throw new BoxError(BoxError.EXTERNAL_ERROR, `Malformed update: ${response.status} ${response.text}`); throw new BoxError(BoxError.EXTERNAL_ERROR, `Malformed update: ${response.status} ${response.text}`);
} }
@@ -268,7 +268,7 @@ async function updateCloudron(data) {
if (response.status === 401) throw new BoxError(BoxError.LICENSE_ERROR, 'Invalid appstore token'); if (response.status === 401) throw new BoxError(BoxError.LICENSE_ERROR, 'Invalid appstore token');
if (response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, `Bad response: ${response.status} ${response.text}`); if (response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, `Bad response: ${response.status} ${response.text}`);
debug(`updateCloudron: Cloudron updated with data ${JSON.stringify(data)}`); log(`updateCloudron: Cloudron updated with data ${JSON.stringify(data)}`);
} }
async function registerCloudron3() { async function registerCloudron3() {
@@ -277,7 +277,7 @@ async function registerCloudron3() {
const token = await settings.get(settings.APPSTORE_API_TOKEN_KEY); const token = await settings.get(settings.APPSTORE_API_TOKEN_KEY);
if (token) { // when installed using setupToken, this updates the domain record when called during provisioning if (token) { // when installed using setupToken, this updates the domain record when called during provisioning
debug('registerCloudron3: already registered. Just updating the record.'); log('registerCloudron3: already registered. Just updating the record.');
await getSubscription(); await getSubscription();
return await updateCloudron({ domain, version }); return await updateCloudron({ domain, version });
} }
@@ -296,7 +296,7 @@ async function registerCloudron3() {
await settings.set(settings.CLOUDRON_ID_KEY, response.body.cloudronId); await settings.set(settings.CLOUDRON_ID_KEY, response.body.cloudronId);
await settings.set(settings.APPSTORE_API_TOKEN_KEY, response.body.cloudronToken); await settings.set(settings.APPSTORE_API_TOKEN_KEY, response.body.cloudronToken);
debug(`registerCloudron3: Cloudron registered with id ${response.body.cloudronId}`); log(`registerCloudron3: Cloudron registered with id ${response.body.cloudronId}`);
await getSubscription(); await getSubscription();
} }
@@ -307,7 +307,7 @@ async function unregister() {
} }
async function unlinkAccount() { async function unlinkAccount() {
debug('unlinkAccount: Unlinking existing account.'); log('unlinkAccount: Unlinking existing account.');
if (constants.DEMO) throw new BoxError(BoxError.BAD_STATE, 'Not allowed in demo mode'); if (constants.DEMO) throw new BoxError(BoxError.BAD_STATE, 'Not allowed in demo mode');
@@ -326,7 +326,7 @@ async function downloadManifest(appStoreId, manifest) {
const url = await getApiServerOrigin() + '/api/v1/apps/' + id + (version ? '/versions/' + version : ''); const url = await getApiServerOrigin() + '/api/v1/apps/' + id + (version ? '/versions/' + version : '');
debug(`downloading manifest from ${url}`); log(`downloading manifest from ${url}`);
const [error, response] = await safe(superagent.get(url).timeout(60 * 1000).ok(() => true)); const [error, response] = await safe(superagent.get(url).timeout(60 * 1000).ok(() => true));
@@ -388,7 +388,7 @@ async function getApp(appId) {
async function downloadIcon(appStoreId, version) { async function downloadIcon(appStoreId, version) {
const iconUrl = `${await getApiServerOrigin()}/api/v1/apps/${appStoreId}/versions/${version}/icon`; const iconUrl = `${await getApiServerOrigin()}/api/v1/apps/${appStoreId}/versions/${version}/icon`;
return await promiseRetry({ times: 10, interval: 5000, debug }, async function () { return await promiseRetry({ times: 10, interval: 5000, debug: log }, async function () {
const [networkError, response] = await safe(superagent.get(iconUrl) const [networkError, response] = await safe(superagent.get(iconUrl)
.timeout(60 * 1000) .timeout(60 * 1000)
.ok(() => true)); .ok(() => true));

34
src/apptask.js
View File

@@ -9,7 +9,7 @@ import backuptask from './backuptask.js';
import BoxError from './boxerror.js'; import BoxError from './boxerror.js';
import community from './community.js'; import community from './community.js';
import constants from './constants.js'; import constants from './constants.js';
import debugModule from 'debug'; import logger from './logger.js';
import df from './df.js'; import df from './df.js';
import dns from './dns.js'; import dns from './dns.js';
import docker from './docker.js'; import docker from './docker.js';
@@ -28,7 +28,7 @@ import services from './services.js';
import shellModule from './shell.js'; import shellModule from './shell.js';
import _ from './underscore.js'; import _ from './underscore.js';
const debug = debugModule('box:apptask'); const { log, trace } = logger('apptask');
const shell = shellModule('apptask'); const shell = shellModule('apptask');
const LOGROTATE_CONFIG_EJS = fs.readFileSync(import.meta.dirname + '/logrotate.ejs', { encoding: 'utf8' }), const LOGROTATE_CONFIG_EJS = fs.readFileSync(import.meta.dirname + '/logrotate.ejs', { encoding: 'utf8' }),
@@ -63,7 +63,7 @@ async function allocateContainerIp(app) {
if (app.manifest.id === constants.PROXY_APP_APPSTORE_ID) return; if (app.manifest.id === constants.PROXY_APP_APPSTORE_ID) return;
await promiseRetry({ times: 10, interval: 0, debug }, async function () { await promiseRetry({ times: 10, interval: 0, debug: log }, async function () {
const iprange = iputils.intFromIp(constants.APPS_IPv4_END) - iputils.intFromIp(constants.APPS_IPv4_START); const iprange = iputils.intFromIp(constants.APPS_IPv4_END) - iputils.intFromIp(constants.APPS_IPv4_START);
const rnd = Math.floor(Math.random() * iprange); const rnd = Math.floor(Math.random() * iprange);
const containerIp = iputils.ipFromInt(iputils.intFromIp(constants.APPS_IPv4_START) + rnd); const containerIp = iputils.ipFromInt(iputils.intFromIp(constants.APPS_IPv4_START) + rnd);
@@ -92,7 +92,7 @@ async function deleteAppDir(app, options) {
const resolvedAppDataDir = stat.isSymbolicLink() ? safe.fs.readlinkSync(appDataDir) : appDataDir; const resolvedAppDataDir = stat.isSymbolicLink() ? safe.fs.readlinkSync(appDataDir) : appDataDir;
debug(`deleteAppDir - removing files in ${resolvedAppDataDir}`); log(`deleteAppDir - removing files in ${resolvedAppDataDir}`);
if (safe.fs.existsSync(resolvedAppDataDir)) { if (safe.fs.existsSync(resolvedAppDataDir)) {
const entries = safe.fs.readdirSync(resolvedAppDataDir); const entries = safe.fs.readdirSync(resolvedAppDataDir);
@@ -105,7 +105,7 @@ async function deleteAppDir(app, options) {
const entryStat = safe.fs.statSync(fullPath); const entryStat = safe.fs.statSync(fullPath);
if (entryStat && !entryStat.isDirectory()) { if (entryStat && !entryStat.isDirectory()) {
safe.fs.unlinkSync(fullPath); safe.fs.unlinkSync(fullPath);
debug(`deleteAppDir - ${fullPath} ${safe.error?.message || ''}`); log(`deleteAppDir - ${fullPath} ${safe.error?.message || ''}`);
} }
} }
} }
@@ -135,7 +135,7 @@ async function deleteContainers(app, options) {
assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof app, 'object');
assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof options, 'object');
debug('deleteContainer: deleting app containers (app, scheduler)'); log('deleteContainer: deleting app containers (app, scheduler)');
// remove configs that rely on container id // remove configs that rely on container id
await removeLogrotateConfig(app); await removeLogrotateConfig(app);
@@ -149,7 +149,7 @@ async function cleanupLogs(app) {
// note that redis container logs are cleaned up by the addon // note that redis container logs are cleaned up by the addon
const [error] = await safe(fs.promises.rm(path.join(paths.LOG_DIR, app.id), { force: true, recursive: true })); const [error] = await safe(fs.promises.rm(path.join(paths.LOG_DIR, app.id), { force: true, recursive: true }));
if (error) debug('cleanupLogs: cannot cleanup logs: %o', error); if (error) log('cleanupLogs: cannot cleanup logs: %o', error);
} }
async function verifyManifest(manifest) { async function verifyManifest(manifest) {
@@ -167,10 +167,10 @@ async function downloadIcon(app) {
let packageIcon = null; let packageIcon = null;
if (app.versionsUrl && app.manifest.iconUrl) { if (app.versionsUrl && app.manifest.iconUrl) {
debug(`downloadIcon: Downloading community icon ${app.manifest.iconUrl}`); log(`downloadIcon: Downloading community icon ${app.manifest.iconUrl}`);
packageIcon = await community.downloadIcon(app.manifest); packageIcon = await community.downloadIcon(app.manifest);
} else if (app.appStoreId) { } else if (app.appStoreId) {
debug(`downloadIcon: Downloading icon of ${app.appStoreId}@${app.manifest.version}`); log(`downloadIcon: Downloading icon of ${app.appStoreId}@${app.manifest.version}`);
packageIcon = await appstore.downloadIcon(app.appStoreId, app.manifest.version); packageIcon = await appstore.downloadIcon(app.appStoreId, app.manifest.version);
} }
@@ -251,7 +251,7 @@ async function updateChecklist(app, newChecks, acknowledged = false) {
} }
async function startApp(app) { async function startApp(app) {
debug('startApp: starting container'); log('startApp: starting container');
if (app.runState === apps.RSTATE_STOPPED) return; if (app.runState === apps.RSTATE_STOPPED) return;
@@ -378,7 +378,7 @@ async function createContainer(app) {
if (app.manifest.id === constants.PROXY_APP_APPSTORE_ID) return; if (app.manifest.id === constants.PROXY_APP_APPSTORE_ID) return;
debug('createContainer: creating container'); log('createContainer: creating container');
const container = await docker.createContainer(app); const container = await docker.createContainer(app);
@@ -808,7 +808,7 @@ async function run(appId, args, progressCallback) {
const app = await apps.get(appId); const app = await apps.get(appId);
debug(`run: startTask installationState: ${app.installationState} runState: ${app.runState}`); log(`run: startTask installationState: ${app.installationState} runState: ${app.runState}`);
let cmd; let cmd;
@@ -855,19 +855,19 @@ async function run(appId, args, progressCallback) {
cmd = updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null }); cmd = updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null });
break; break;
default: default:
debug('run: apptask launched with invalid command'); log('run: apptask launched with invalid command');
throw new BoxError(BoxError.INTERNAL_ERROR, 'Unknown install command in apptask:' + app.installationState); throw new BoxError(BoxError.INTERNAL_ERROR, 'Unknown install command in apptask:' + app.installationState);
} }
const [error, result] = await safe(cmd); // only some commands like backup return a result const [error, result] = await safe(cmd); // only some commands like backup return a result
if (error) { if (error) {
debug(`run: app error for state ${app.installationState}: %o`, error); log(`run: app error for state ${app.installationState}: %o`, error);
if (app.installationState === apps.ISTATE_PENDING_UPDATE && error.backupError) { if (app.installationState === apps.ISTATE_PENDING_UPDATE && error.backupError) {
debug('run: update aborted because backup failed'); log('run: update aborted because backup failed');
await safe(updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null }, { debug })); await safe(updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null }, { debug: log }));
} else { } else {
await safe(updateApp(app, { installationState: apps.ISTATE_ERROR, error: makeTaskError(error, app) }), { debug }); await safe(updateApp(app, { installationState: apps.ISTATE_ERROR, error: makeTaskError(error, app) }), { debug: log });
} }
throw error; throw error;

12
src/apptaskmanager.js
View File

@@ -1,6 +1,6 @@
import assert from 'node:assert'; import assert from 'node:assert';
import BoxError from './boxerror.js'; import BoxError from './boxerror.js';
import debugModule from 'debug'; import logger from './logger.js';
import fs from 'node:fs'; import fs from 'node:fs';
import locks from './locks.js'; import locks from './locks.js';
import path from 'node:path'; import path from 'node:path';
@@ -9,7 +9,7 @@ import safe from 'safetydance';
import scheduler from './scheduler.js'; import scheduler from './scheduler.js';
import tasks from './tasks.js'; import tasks from './tasks.js';
const debug = debugModule('box:apptaskmanager'); const { log, trace } = logger('apptaskmanager');
const gActiveTasks = {}; // indexed by app id const gActiveTasks = {}; // indexed by app id
@@ -22,12 +22,12 @@ const DRAIN_TIMER_SECS = 1000;
let gDrainTimerId = null; let gDrainTimerId = null;
async function drain() { async function drain() {
debug(`drain: ${gPendingTasks.length} apptasks pending`); log(`drain: ${gPendingTasks.length} apptasks pending`);
for (let i = 0; i < gPendingTasks.length; i++) { for (let i = 0; i < gPendingTasks.length; i++) {
const space = Object.keys(gActiveTasks).length - TASK_CONCURRENCY; const space = Object.keys(gActiveTasks).length - TASK_CONCURRENCY;
if (space == 0) { if (space == 0) {
debug('At concurrency limit, cannot drain anymore'); log('At concurrency limit, cannot drain anymore');
break; break;
} }
@@ -53,7 +53,7 @@ async function drain() {
.catch((error) => { taskError = error; }) .catch((error) => { taskError = error; })
.finally(async () => { .finally(async () => {
delete gActiveTasks[appId]; delete gActiveTasks[appId];
await safe(onFinished(taskError, taskResult), { debug }); // hasPendingTasks() can now return false await safe(onFinished(taskError, taskResult), { debug: log }); // hasPendingTasks() can now return false
await locks.release(`${locks.TYPE_APP_TASK_PREFIX}${appId}`); await locks.release(`${locks.TYPE_APP_TASK_PREFIX}${appId}`);
await locks.releaseByTaskId(taskId); await locks.releaseByTaskId(taskId);
scheduler.resumeAppJobs(appId); scheduler.resumeAppJobs(appId);
@@ -68,7 +68,7 @@ async function start() {
assert.strictEqual(gDrainTimerId, null); assert.strictEqual(gDrainTimerId, null);
assert.strictEqual(gStarted, false); assert.strictEqual(gStarted, false);
debug('started'); log('started');
gStarted = true; gStarted = true;
if (gPendingTasks.length) gDrainTimerId = setTimeout(drain, DRAIN_TIMER_SECS); if (gPendingTasks.length) gDrainTimerId = setTimeout(drain, DRAIN_TIMER_SECS);

10
src/asynctask.js
View File

@@ -1,8 +1,8 @@
import debugModule from 'debug'; import logger from './logger.js';
import EventEmitter from 'node:events'; import EventEmitter from 'node:events';
import safe from 'safetydance'; import safe from 'safetydance';
const debug = debugModule('box:asynctask'); const { log, trace } = logger('asynctask');
// this runs in-process // this runs in-process
class AsyncTask extends EventEmitter { class AsyncTask extends EventEmitter {
@@ -20,16 +20,16 @@ class AsyncTask extends EventEmitter {
} }
async start() { // should not throw! async start() { // should not throw!
debug(`start: ${this.name} started`); log(`start: ${this.name} started`);
const [error] = await safe(this._run(this.#abortController.signal)); const [error] = await safe(this._run(this.#abortController.signal));
debug(`start: ${this.name} finished`); log(`start: ${this.name} finished`);
this.emit('done', { errorMessage: error?.message || '' }); this.emit('done', { errorMessage: error?.message || '' });
this.#abortController = null; this.#abortController = null;
} }
stop() { stop() {
if (this.#abortController === null) return; // already finished if (this.#abortController === null) return; // already finished
debug(`stop: ${this.name} . sending abort signal`); log(`stop: ${this.name} . sending abort signal`);
this.#abortController.abort(); this.#abortController.abort();
} }

44
src/backupcleaner.js
View File

@@ -6,12 +6,12 @@ import backups from './backups.js';
import backupFormats from './backupformats.js'; import backupFormats from './backupformats.js';
import backupSites from './backupsites.js'; import backupSites from './backupsites.js';
import constants from './constants.js'; import constants from './constants.js';
import debugModule from 'debug'; import logger from './logger.js';
import moment from 'moment'; import moment from 'moment';
import path from 'node:path'; import path from 'node:path';
import safe from 'safetydance'; import safe from 'safetydance';
const debug = debugModule('box:backupcleaner'); const { log, trace } = logger('backupcleaner');
function applyBackupRetention(allBackups, retention, referencedBackupIds) { function applyBackupRetention(allBackups, retention, referencedBackupIds) {
assert(Array.isArray(allBackups)); assert(Array.isArray(allBackups));
@@ -67,7 +67,7 @@ function applyBackupRetention(allBackups, retention, referencedBackupIds) {
} }
for (const backup of allBackups) { for (const backup of allBackups) {
debug(`applyBackupRetention: ${backup.remotePath} keep/discard: ${backup.keepReason || backup.discardReason || 'unprocessed'}`); log(`applyBackupRetention: ${backup.remotePath} keep/discard: ${backup.keepReason || backup.discardReason || 'unprocessed'}`);
} }
} }
@@ -88,21 +88,21 @@ async function removeBackup(site, backup, progressCallback) {
} }
if (removeError) { if (removeError) {
debug(`removeBackup: error removing backup ${removeError.message}`); log(`removeBackup: error removing backup ${removeError.message}`);
return; return;
} }
// remove integrity info // remove integrity info
const [removeIntegrityError] = await safe(backupSites.storageApi(site).remove(site.config, `${remotePath}.backupinfo`)); const [removeIntegrityError] = await safe(backupSites.storageApi(site).remove(site.config, `${remotePath}.backupinfo`));
if (removeIntegrityError) debug(`removeBackup: could not remove integrity info: ${removeIntegrityError.message}`); if (removeIntegrityError) log(`removeBackup: could not remove integrity info: ${removeIntegrityError.message}`);
// prune empty directory if possible // prune empty directory if possible
const [pruneError] = await safe(backupSites.storageApi(site).remove(site.config, path.dirname(remotePath))); const [pruneError] = await safe(backupSites.storageApi(site).remove(site.config, path.dirname(remotePath)));
if (pruneError) debug(`removeBackup: unable to prune backup directory ${path.dirname(remotePath)}: ${pruneError.message}`); if (pruneError) log(`removeBackup: unable to prune backup directory ${path.dirname(remotePath)}: ${pruneError.message}`);
const [delError] = await safe(backups.del(backup.id)); const [delError] = await safe(backups.del(backup.id));
if (delError) debug(`removeBackup: error removing ${backup.id} from database. %o`, delError); if (delError) log(`removeBackup: error removing ${backup.id} from database. %o`, delError);
else debug(`removeBackup: removed ${backup.remotePath}`); else log(`removeBackup: removed ${backup.remotePath}`);
} }
async function cleanupAppBackups(site, referencedBackupIds, progressCallback) { async function cleanupAppBackups(site, referencedBackupIds, progressCallback) {
@@ -129,7 +129,7 @@ async function cleanupAppBackups(site, referencedBackupIds, progressCallback) {
let appBackupsToRemove = []; let appBackupsToRemove = [];
for (const appId of Object.keys(appBackupsById)) { for (const appId of Object.keys(appBackupsById)) {
const appRetention = Object.assign({ keepLatest: allAppIds.includes(appId) }, site.retention); const appRetention = Object.assign({ keepLatest: allAppIds.includes(appId) }, site.retention);
debug(`cleanupAppBackups: applying retention for appId ${appId} retention: ${JSON.stringify(appRetention)}`); log(`cleanupAppBackups: applying retention for appId ${appId} retention: ${JSON.stringify(appRetention)}`);
applyBackupRetention(appBackupsById[appId], appRetention, referencedBackupIds); applyBackupRetention(appBackupsById[appId], appRetention, referencedBackupIds);
appBackupsToRemove = appBackupsToRemove.concat(appBackupsById[appId].filter(b => !b.keepReason)); appBackupsToRemove = appBackupsToRemove.concat(appBackupsById[appId].filter(b => !b.keepReason));
} }
@@ -140,7 +140,7 @@ async function cleanupAppBackups(site, referencedBackupIds, progressCallback) {
await removeBackup(site, appBackup, progressCallback); // never errors await removeBackup(site, appBackup, progressCallback); // never errors
} }
debug('cleanupAppBackups: done'); log('cleanupAppBackups: done');
return removedAppBackupPaths; return removedAppBackupPaths;
} }
@@ -163,7 +163,7 @@ async function cleanupMailBackups(site, referencedBackupIds, progressCallback) {
await removeBackup(site, mailBackup, progressCallback); // never errors await removeBackup(site, mailBackup, progressCallback); // never errors
} }
debug('cleanupMailBackups: done'); log('cleanupMailBackups: done');
return removedMailBackupPaths; return removedMailBackupPaths;
} }
@@ -193,7 +193,7 @@ async function cleanupBoxBackups(site, progressCallback) {
await removeBackup(site, boxBackup, progressCallback); await removeBackup(site, boxBackup, progressCallback);
} }
debug('cleanupBoxBackups: done'); log('cleanupBoxBackups: done');
return { removedBoxBackupPaths, referencedBackupIds }; return { removedBoxBackupPaths, referencedBackupIds };
} }
@@ -223,7 +223,7 @@ async function cleanupMissingBackups(site, progressCallback) {
await progressCallback({ message: `Removing missing backup ${backup.remotePath}`}); await progressCallback({ message: `Removing missing backup ${backup.remotePath}`});
const [delError] = await safe(backups.del(backup.id)); const [delError] = await safe(backups.del(backup.id));
if (delError) debug(`cleanupMissingBackups: error removing ${backup.id} from database. %o`, delError); if (delError) log(`cleanupMissingBackups: error removing ${backup.id} from database. %o`, delError);
missingBackupPaths.push(backup.remotePath); missingBackupPaths.push(backup.remotePath);
} }
@@ -231,7 +231,7 @@ async function cleanupMissingBackups(site, progressCallback) {
++ page; ++ page;
} while (result.length === perPage); } while (result.length === perPage);
debug('cleanupMissingBackups: done'); log('cleanupMissingBackups: done');
return missingBackupPaths; return missingBackupPaths;
} }
@@ -242,7 +242,7 @@ async function removeOldAppSnapshots(site) {
const snapshotInfo = await backupSites.getSnapshotInfo(site); const snapshotInfo = await backupSites.getSnapshotInfo(site);
const progressCallback = (progress) => { debug(`removeOldAppSnapshots: ${progress.message}`); }; const progressCallback = (progress) => { log(`removeOldAppSnapshots: ${progress.message}`); };
for (const appId of Object.keys(snapshotInfo)) { for (const appId of Object.keys(snapshotInfo)) {
if (appId === 'box' || appId === 'mail') continue; if (appId === 'box' || appId === 'mail') continue;
@@ -253,16 +253,16 @@ async function removeOldAppSnapshots(site) {
const ext = backupFormats.api(site.format).getFileExtension(!!site.encryption); const ext = backupFormats.api(site.format).getFileExtension(!!site.encryption);
const remotePath = `snapshot/app_${appId}${ext}`; const remotePath = `snapshot/app_${appId}${ext}`;
if (ext) { if (ext) {
await safe(backupSites.storageApi(site).remove(site.config, remotePath), { debug }); await safe(backupSites.storageApi(site).remove(site.config, remotePath), { debug: log });
} else { } else {
await safe(backupSites.storageApi(site).removeDir(site.config, site.limits, remotePath, progressCallback), { debug }); await safe(backupSites.storageApi(site).removeDir(site.config, site.limits, remotePath, progressCallback), { debug: log });
} }
await backupSites.setSnapshotInfo(site, appId, null /* info */); await backupSites.setSnapshotInfo(site, appId, null /* info */);
debug(`removeOldAppSnapshots: removed snapshot of app ${appId}`); log(`removeOldAppSnapshots: removed snapshot of app ${appId}`);
} }
debug('removeOldAppSnapshots: done'); log('removeOldAppSnapshots: done');
} }
async function run(siteId, progressCallback) { async function run(siteId, progressCallback) {
@@ -272,14 +272,14 @@ async function run(siteId, progressCallback) {
const site = await backupSites.get(siteId); const site = await backupSites.get(siteId);
if (!site) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Target not found'); if (!site) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Target not found');
debug(`run: retention is ${JSON.stringify(site.retention)}`); log(`run: retention is ${JSON.stringify(site.retention)}`);
const status = await backupSites.ensureMounted(site); const status = await backupSites.ensureMounted(site);
debug(`run: mount point status is ${JSON.stringify(status)}`); log(`run: mount point status is ${JSON.stringify(status)}`);
if (status.state !== 'active') throw new BoxError(BoxError.MOUNT_ERROR, `Backup endpoint is not mounted: ${status.message}`); if (status.state !== 'active') throw new BoxError(BoxError.MOUNT_ERROR, `Backup endpoint is not mounted: ${status.message}`);
if (site.retention.keepWithinSecs < 0) { if (site.retention.keepWithinSecs < 0) {
debug('run: keeping all backups'); log('run: keeping all backups');
return {}; return {};
} }

40
src/backupformat/rsync.js
View File

@@ -3,7 +3,7 @@ import async from 'async';
import backupSites from '../backupsites.js'; import backupSites from '../backupsites.js';
import BoxError from '../boxerror.js'; import BoxError from '../boxerror.js';
import DataLayout from '../datalayout.js'; import DataLayout from '../datalayout.js';
import debugModule from 'debug'; import logger from '../logger.js';
import hush from '../hush.js'; import hush from '../hush.js';
const { DecryptStream, EncryptStream } = hush; const { DecryptStream, EncryptStream } = hush;
import fs from 'node:fs'; import fs from 'node:fs';
@@ -19,7 +19,7 @@ import syncer from '../syncer.js';
import util from 'node:util'; import util from 'node:util';
import { Writable } from 'node:stream'; import { Writable } from 'node:stream';
const debug = debugModule('box:backupformat/rsync'); const { log, trace } = logger('backupformat/rsync');
const shell = shellModule('backupformat/rsync'); const shell = shellModule('backupformat/rsync');
async function addFile(sourceFile, encryption, uploader, progressCallback) { async function addFile(sourceFile, encryption, uploader, progressCallback) {
@@ -32,7 +32,7 @@ async function addFile(sourceFile, encryption, uploader, progressCallback) {
// destinations dirs/file which are owned by root (this process id) and cannot be copied (run as normal user) // destinations dirs/file which are owned by root (this process id) and cannot be copied (run as normal user)
const [openError, sourceHandle] = await safe(fs.promises.open(sourceFile, 'r')); const [openError, sourceHandle] = await safe(fs.promises.open(sourceFile, 'r'));
if (openError) { if (openError) {
debug(`addFile: ignoring disappeared file: ${sourceFile}`); log(`addFile: ignoring disappeared file: ${sourceFile}`);
return { integrity: null, stats: { transferred: 0 } }; return { integrity: null, stats: { transferred: 0 } };
} }
@@ -61,7 +61,7 @@ async function addFile(sourceFile, encryption, uploader, progressCallback) {
const [error] = await safe(pipelinePromise); const [error] = await safe(pipelinePromise);
if (error && !error.message.includes('ENOENT')) throw new BoxError(BoxError.EXTERNAL_ERROR, `tarPack pipeline error for ${sourceFile}: ${error.message}`); // ignore error if file disappears if (error && !error.message.includes('ENOENT')) throw new BoxError(BoxError.EXTERNAL_ERROR, `tarPack pipeline error for ${sourceFile}: ${error.message}`); // ignore error if file disappears
// debug(`addFile: pipeline finished: ${JSON.stringify(ps.stats())}`); // log(`addFile: pipeline finished: ${JSON.stringify(ps.stats())}`);
await uploader.finish(); await uploader.finish();
@@ -112,7 +112,7 @@ async function restoreFsMetadata(dataLayout, metadataFile) {
assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout'); assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
assert.strictEqual(typeof metadataFile, 'string'); assert.strictEqual(typeof metadataFile, 'string');
debug(`Recreating empty directories in ${dataLayout.toString()}`); log(`Recreating empty directories in ${dataLayout.toString()}`);
const metadataJson = safe.fs.readFileSync(metadataFile, 'utf8'); const metadataJson = safe.fs.readFileSync(metadataFile, 'utf8');
if (metadataJson === null) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Error loading fsmetadata.json:' + safe.error.message); if (metadataJson === null) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Error loading fsmetadata.json:' + safe.error.message);
@@ -164,7 +164,7 @@ async function sync(backupSite, remotePath, dataLayout, progressCallback) {
const concurrency = backupSite.limits?.syncConcurrency || (backupSite.provider === 's3' ? 20 : 10); const concurrency = backupSite.limits?.syncConcurrency || (backupSite.provider === 's3' ? 20 : 10);
const cacheFile = path.join(paths.BACKUP_INFO_DIR, backupSite.id, `${dataLayout.getBasename()}.sync.cache`); const cacheFile = path.join(paths.BACKUP_INFO_DIR, backupSite.id, `${dataLayout.getBasename()}.sync.cache`);
const { delQueue, addQueue, integrityMap } = await syncer.sync(dataLayout, cacheFile); // integrityMap is unchanged files const { delQueue, addQueue, integrityMap } = await syncer.sync(dataLayout, cacheFile); // integrityMap is unchanged files
debug(`sync: processing ${delQueue.length} deletes, ${addQueue.length} additions and ${integrityMap.size} unchanged`); log(`sync: processing ${delQueue.length} deletes, ${addQueue.length} additions and ${integrityMap.size} unchanged`);
const aggregatedStats = { const aggregatedStats = {
transferred: 0, transferred: 0,
size: [...integrityMap.values()].reduce((sum, integrity) => sum + (integrity?.size || 0), 0), // integrity can be null if file had disappeared during upload size: [...integrityMap.values()].reduce((sum, integrity) => sum + (integrity?.size || 0), 0), // integrity can be null if file had disappeared during upload
@@ -193,9 +193,9 @@ async function sync(backupSite, remotePath, dataLayout, progressCallback) {
} else if (change.operation === 'remove') { } else if (change.operation === 'remove') {
await backupSites.storageApi(backupSite).remove(backupSite.config, fullPath); await backupSites.storageApi(backupSite).remove(backupSite.config, fullPath);
} else if (change.operation === 'add') { } else if (change.operation === 'add') {
await promiseRetry({ times: 5, interval: 20000, debug }, async (retryCount) => { await promiseRetry({ times: 5, interval: 20000, debug: log }, async (retryCount) => {
reportUploadProgress(`Current: ${change.path}` + (retryCount > 1 ? ` (Try ${retryCount})` : '')); reportUploadProgress(`Current: ${change.path}` + (retryCount > 1 ? ` (Try ${retryCount})` : ''));
if (retryCount > 1) debug(`sync: retrying ${change.path} position ${change.position} try ${retryCount}`); if (retryCount > 1) log(`sync: retrying ${change.path} position ${change.position} try ${retryCount}`);
const uploader = await backupSites.storageApi(backupSite).upload(backupSite.config, backupSite.limits, fullPath); const uploader = await backupSites.storageApi(backupSite).upload(backupSite.config, backupSite.limits, fullPath);
const result = await addFile(dataLayout.toLocalPath('./' + change.path), backupSite.encryption, uploader, (progress) => { const result = await addFile(dataLayout.toLocalPath('./' + change.path), backupSite.encryption, uploader, (progress) => {
reportUploadProgress(progress.message); reportUploadProgress(progress.message);
@@ -210,11 +210,11 @@ async function sync(backupSite, remotePath, dataLayout, progressCallback) {
} }
const [delError] = await safe(async.eachLimit(delQueue, concurrency, async (change) => await processSyncerChange(change, backupSite, remotePath, dataLayout, progressCallback))); const [delError] = await safe(async.eachLimit(delQueue, concurrency, async (change) => await processSyncerChange(change, backupSite, remotePath, dataLayout, progressCallback)));
debug('sync: done processing deletes. error: %o', delError); log('sync: done processing deletes. error: %o', delError);
if (delError) throw delError; if (delError) throw delError;
const [addError] = await safe(async.eachLimit(addQueue, concurrency, async (change) => await processSyncerChange(change, backupSite, remotePath, dataLayout, progressCallback))); const [addError] = await safe(async.eachLimit(addQueue, concurrency, async (change) => await processSyncerChange(change, backupSite, remotePath, dataLayout, progressCallback)));
debug('sync: done processing adds. error: %o', addError); log('sync: done processing adds. error: %o', addError);
if (addError) throw addError; if (addError) throw addError;
progressCallback({ message: `Uploaded ${completedAdds} files` }); progressCallback({ message: `Uploaded ${completedAdds} files` });
@@ -235,7 +235,7 @@ async function downloadDir(backupSite, remotePath, dataLayout, progressCallback)
const encryptedFilenames = backupSite.encryption?.encryptedFilenames || false; const encryptedFilenames = backupSite.encryption?.encryptedFilenames || false;
debug(`downloadDir: ${remotePath} to ${dataLayout.toString()}. encryption filenames: ${encryptedFilenames}. encrypted files: ${!!backupSite.encryption}`); log(`downloadDir: ${remotePath} to ${dataLayout.toString()}. encryption filenames: ${encryptedFilenames}. encrypted files: ${!!backupSite.encryption}`);
let completedFiles = 0, totalFiles = 0; let completedFiles = 0, totalFiles = 0;
let lastProgressTime = 0; let lastProgressTime = 0;
@@ -264,7 +264,7 @@ async function downloadDir(backupSite, remotePath, dataLayout, progressCallback)
const [downloadError, sourceStream] = await safe(backupSites.storageApi(backupSite).download(backupSite.config, entry.path)); const [downloadError, sourceStream] = await safe(backupSites.storageApi(backupSite).download(backupSite.config, entry.path));
if (downloadError) { if (downloadError) {
debug(`downloadDir: download ${entry.path} to ${destFilePath} errored: ${downloadError.message}`); log(`downloadDir: download ${entry.path} to ${destFilePath} errored: ${downloadError.message}`);
throw downloadError; throw downloadError;
} }
@@ -290,7 +290,7 @@ async function downloadDir(backupSite, remotePath, dataLayout, progressCallback)
const [pipelineError] = await safe(pipeline(streams)); const [pipelineError] = await safe(pipeline(streams));
if (pipelineError) { if (pipelineError) {
debug(`downloadDir: download error ${entry.path} to ${destFilePath}: ${pipelineError.message}`); log(`downloadDir: download error ${entry.path} to ${destFilePath}: ${pipelineError.message}`);
throw pipelineError; throw pipelineError;
} }
}); });
@@ -318,7 +318,7 @@ async function download(backupSite, remotePath, dataLayout, progressCallback) {
assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout'); assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
assert.strictEqual(typeof progressCallback, 'function'); assert.strictEqual(typeof progressCallback, 'function');
debug(`download: Downloading ${remotePath} to ${dataLayout.toString()}`); log(`download: Downloading ${remotePath} to ${dataLayout.toString()}`);
await downloadDir(backupSite, remotePath, dataLayout, progressCallback); await downloadDir(backupSite, remotePath, dataLayout, progressCallback);
await restoreFsMetadata(dataLayout, `${dataLayout.localRoot()}/fsmetadata.json`); await restoreFsMetadata(dataLayout, `${dataLayout.localRoot()}/fsmetadata.json`);
@@ -330,7 +330,7 @@ async function upload(backupSite, remotePath, dataLayout, progressCallback) {
assert.strictEqual(typeof dataLayout, 'object'); assert.strictEqual(typeof dataLayout, 'object');
assert.strictEqual(typeof progressCallback, 'function'); assert.strictEqual(typeof progressCallback, 'function');
debug(`upload: uploading to site ${backupSite.id} path ${remotePath} (encrypted: ${!!backupSite.encryption}) dataLayout ${dataLayout.toString()}`); log(`upload: uploading to site ${backupSite.id} path ${remotePath} (encrypted: ${!!backupSite.encryption}) dataLayout ${dataLayout.toString()}`);
await saveFsMetadata(dataLayout, `${dataLayout.localRoot()}/fsmetadata.json`); await saveFsMetadata(dataLayout, `${dataLayout.localRoot()}/fsmetadata.json`);
return await sync(backupSite, remotePath, dataLayout, progressCallback); // { stats, integrityMap } return await sync(backupSite, remotePath, dataLayout, progressCallback); // { stats, integrityMap }
@@ -370,7 +370,7 @@ async function verify(backupSite, remotePath, integrityMap, progressCallback) {
assert(util.types.isMap(integrityMap), 'integrityMap should be a Map'); assert(util.types.isMap(integrityMap), 'integrityMap should be a Map');
assert.strictEqual(typeof progressCallback, 'function'); assert.strictEqual(typeof progressCallback, 'function');
debug(`verify: Verifying ${remotePath}`); log(`verify: Verifying ${remotePath}`);
// https://www.digitalocean.com/community/questions/rate-limiting-on-spaces?answer=40441 // https://www.digitalocean.com/community/questions/rate-limiting-on-spaces?answer=40441
const concurrency = backupSite.limits?.downloadConcurrency || (backupSite.provider === 's3' ? 30 : 10); const concurrency = backupSite.limits?.downloadConcurrency || (backupSite.provider === 's3' ? 30 : 10);
@@ -386,12 +386,12 @@ async function verify(backupSite, remotePath, integrityMap, progressCallback) {
const integrity = integrityMap.get(relativePath); const integrity = integrityMap.get(relativePath);
if (result.transferred !== integrity.size) { if (result.transferred !== integrity.size) {
messages.push(`${entry.path} has size ${result.transferred}. Expecting ${integrity.size}`); messages.push(`${entry.path} has size ${result.transferred}. Expecting ${integrity.size}`);
debug(`verify: size check of ${entry.path} failed: ${messages.at(-1)}`); log(`verify: size check of ${entry.path} failed: ${messages.at(-1)}`);
} else if (result.digest !== integrity.sha256) { } else if (result.digest !== integrity.sha256) {
messages.push(`${entry.path} has digest ${result.digest}. Expecting ${integrity.sha256}`); messages.push(`${entry.path} has digest ${result.digest}. Expecting ${integrity.sha256}`);
debug(`verify: digest check of ${entry.path} failed: ${messages.at(-1)}`); log(`verify: digest check of ${entry.path} failed: ${messages.at(-1)}`);
} else { } else {
debug(`verify: ${entry.path} passed`); log(`verify: ${entry.path} passed`);
} }
}); });
fileCount += batch.entries.length; fileCount += batch.entries.length;
@@ -401,7 +401,7 @@ async function verify(backupSite, remotePath, integrityMap, progressCallback) {
if (integrityMap.size !== fileCount) { if (integrityMap.size !== fileCount) {
messages.push(`Got ${fileCount} files. Expecting ${integrityMap.size} files`); messages.push(`Got ${fileCount} files. Expecting ${integrityMap.size} files`);
debug(`verify: file count mismatch: ${messages.at(-1)}`); log(`verify: file count mismatch: ${messages.at(-1)}`);
} }
return messages; return messages;

56
src/backupformat/tgz.js
View File

@@ -2,7 +2,7 @@ import assert from 'node:assert';
import backupSites from '../backupsites.js'; import backupSites from '../backupsites.js';
import BoxError from '../boxerror.js'; import BoxError from '../boxerror.js';
import DataLayout from '../datalayout.js'; import DataLayout from '../datalayout.js';
import debugModule from 'debug'; import logger from '../logger.js';
import hush from '../hush.js'; import hush from '../hush.js';
const { DecryptStream, EncryptStream } = hush; const { DecryptStream, EncryptStream } = hush;
import fs from 'node:fs'; import fs from 'node:fs';
@@ -17,7 +17,7 @@ import tar from 'tar-stream';
import util from 'node:util'; import util from 'node:util';
import zlib from 'node:zlib'; import zlib from 'node:zlib';
const debug = debugModule('box:backupformat/tgz'); const { log, trace } = logger('backupformat/tgz');
// In tar, the entry header contains the file size. If we don't provide it those many bytes, the tar will become corrupt // In tar, the entry header contains the file size. If we don't provide it those many bytes, the tar will become corrupt
// Linux provides no guarantee of how many bytes can be read from a file. This is the case with sqlite and log files // Linux provides no guarantee of how many bytes can be read from a file. This is the case with sqlite and log files
@@ -31,12 +31,12 @@ class EnsureFileSizeStream extends Transform {
_transform(chunk, encoding, callback) { _transform(chunk, encoding, callback) {
if (this._remaining <= 0) { if (this._remaining <= 0) {
debug(`EnsureFileSizeStream: ${this._name} dropping ${chunk.length} bytes`); log(`EnsureFileSizeStream: ${this._name} dropping ${chunk.length} bytes`);
return callback(null); return callback(null);
} }
if (this._remaining - chunk.length < 0) { if (this._remaining - chunk.length < 0) {
debug(`EnsureFileSizeStream: ${this._name} dropping extra ${chunk.length - this._remaining} bytes`); log(`EnsureFileSizeStream: ${this._name} dropping extra ${chunk.length - this._remaining} bytes`);
chunk = chunk.subarray(0, this._remaining); chunk = chunk.subarray(0, this._remaining);
this._remaining = 0; this._remaining = 0;
} else { } else {
@@ -48,7 +48,7 @@ class EnsureFileSizeStream extends Transform {
_flush(callback) { _flush(callback) {
if (this._remaining > 0) { if (this._remaining > 0) {
debug(`EnsureFileSizeStream: ${this._name} injecting ${this._remaining} bytes`); log(`EnsureFileSizeStream: ${this._name} injecting ${this._remaining} bytes`);
this.push(Buffer.alloc(this._remaining, 0)); this.push(Buffer.alloc(this._remaining, 0));
} }
callback(); callback();
@@ -63,7 +63,7 @@ function addEntryToPack(pack, header, options) {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
const packEntry = safe(() => pack.entry(header, function (error) { const packEntry = safe(() => pack.entry(header, function (error) {
if (error) { if (error) {
debug(`addToPack: error adding ${header.name} ${header.type} ${error.message}`); log(`addToPack: error adding ${header.name} ${header.type} ${error.message}`);
reject(new BoxError(BoxError.FS_ERROR, error.message)); reject(new BoxError(BoxError.FS_ERROR, error.message));
} else { } else {
resolve(); resolve();
@@ -74,7 +74,7 @@ function addEntryToPack(pack, header, options) {
if (options?.input) { if (options?.input) {
const ensureFileSizeStream = new EnsureFileSizeStream({ name: header.name, size: header.size }); const ensureFileSizeStream = new EnsureFileSizeStream({ name: header.name, size: header.size });
safe(stream.pipeline(options.input, ensureFileSizeStream, packEntry), { debug }); // background. rely on pack.entry callback for promise completion safe(stream.pipeline(options.input, ensureFileSizeStream, packEntry), { debug: log }); // background. rely on pack.entry callback for promise completion
} }
}); });
} }
@@ -92,7 +92,7 @@ async function addPathToPack(pack, localPath, dataLayout) {
const dir = queue.shift(); const dir = queue.shift();
const [readdirError, entries] = await safe(fs.promises.readdir(dir, { withFileTypes: true })); const [readdirError, entries] = await safe(fs.promises.readdir(dir, { withFileTypes: true }));
if (!entries) { if (!entries) {
debug(`tarPack: skipping directory ${dir}: ${readdirError.message}`); log(`tarPack: skipping directory ${dir}: ${readdirError.message}`);
continue; continue;
} }
const subdirs = []; const subdirs = [];
@@ -101,9 +101,9 @@ async function addPathToPack(pack, localPath, dataLayout) {
const headerName = dataLayout.toRemotePath(abspath); const headerName = dataLayout.toRemotePath(abspath);
if (entry.isFile()) { if (entry.isFile()) {
const [openError, handle] = await safe(fs.promises.open(abspath, 'r')); const [openError, handle] = await safe(fs.promises.open(abspath, 'r'));
if (!handle) { debug(`tarPack: skipping file, could not open ${abspath}: ${openError.message}`); continue; } if (!handle) { log(`tarPack: skipping file, could not open ${abspath}: ${openError.message}`); continue; }
const [statError, stat] = await safe(handle.stat()); const [statError, stat] = await safe(handle.stat());
if (!stat) { debug(`tarPack: skipping file, could not stat ${abspath}: ${statError.message}`); continue; } if (!stat) { log(`tarPack: skipping file, could not stat ${abspath}: ${statError.message}`); continue; }
const header = { name: headerName, type: 'file', mode: stat.mode, size: stat.size, uid: process.getuid(), gid: process.getgid() }; const header = { name: headerName, type: 'file', mode: stat.mode, size: stat.size, uid: process.getuid(), gid: process.getgid() };
if (stat.size > 8589934590 || entry.name.length > 99) header.pax = { size: stat.size }; if (stat.size > 8589934590 || entry.name.length > 99) header.pax = { size: stat.size };
const input = handle.createReadStream({ autoClose: true }); const input = handle.createReadStream({ autoClose: true });
@@ -116,12 +116,12 @@ async function addPathToPack(pack, localPath, dataLayout) {
++stats.dirCount; ++stats.dirCount;
} else if (entry.isSymbolicLink()) { } else if (entry.isSymbolicLink()) {
const [readlinkError, site] = await safe(fs.promises.readlink(abspath)); const [readlinkError, site] = await safe(fs.promises.readlink(abspath));
if (!site) { debug(`tarPack: skipping link, could not readlink ${abspath}: ${readlinkError.message}`); continue; } if (!site) { log(`tarPack: skipping link, could not readlink ${abspath}: ${readlinkError.message}`); continue; }
const header = { name: headerName, type: 'symlink', linkname: site, uid: process.getuid(), gid: process.getgid() }; const header = { name: headerName, type: 'symlink', linkname: site, uid: process.getuid(), gid: process.getgid() };
await addEntryToPack(pack, header, { /* options */ }); await addEntryToPack(pack, header, { /* options */ });
++stats.linkCount; ++stats.linkCount;
} else { } else {
debug(`tarPack: ignoring unknown type ${entry.name} ${entry.type}`); log(`tarPack: ignoring unknown type ${entry.name} ${entry.type}`);
} }
} }
@@ -163,11 +163,11 @@ async function tarPack(dataLayout, encryption, uploader, progressCallback) {
let fileCount = 0; let fileCount = 0;
for (const localPath of dataLayout.localPaths()) { for (const localPath of dataLayout.localPaths()) {
const [error, stats] = await safe(addPathToPack(pack, localPath, dataLayout), { debug }); const [error, stats] = await safe(addPathToPack(pack, localPath, dataLayout), { debug: log });
if (error) break; // the pipeline will error and we will retry the whole packing all over if (error) break; // the pipeline will error and we will retry the whole packing all over
fileCount += stats.fileCount; fileCount += stats.fileCount;
} }
debug(`tarPack: packed ${fileCount} files`); log(`tarPack: packed ${fileCount} files`);
pack.finalize(); // harmless to call if already in error state pack.finalize(); // harmless to call if already in error state
@@ -175,7 +175,7 @@ async function tarPack(dataLayout, encryption, uploader, progressCallback) {
if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, `tarPack pipeline error: ${error.message}`); if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, `tarPack pipeline error: ${error.message}`);
const stats = ps.stats(); // { startTime, totalMsecs, transferred } const stats = ps.stats(); // { startTime, totalMsecs, transferred }
debug(`tarPack: pipeline finished: ${JSON.stringify(stats)}`); log(`tarPack: pipeline finished: ${JSON.stringify(stats)}`);
await uploader.finish(); await uploader.finish();
return { return {
@@ -195,7 +195,7 @@ async function tarExtract(inStream, dataLayout, encryption, progressCallback) {
let entryCount = 0; let entryCount = 0;
extract.on('entry', async function (header, entryStream, next) { extract.on('entry', async function (header, entryStream, next) {
if (path.isAbsolute(header.name)) { if (path.isAbsolute(header.name)) {
debug(`tarExtract: ignoring absolute path ${header.name}`); log(`tarExtract: ignoring absolute path ${header.name}`);
return next(); return next();
} }
++entryCount; ++entryCount;
@@ -211,7 +211,7 @@ async function tarExtract(inStream, dataLayout, encryption, progressCallback) {
await safe(fs.promises.unlink(abspath)); // remove any link created from previous failed extract await safe(fs.promises.unlink(abspath)); // remove any link created from previous failed extract
[error] = await safe(fs.promises.symlink(header.linkname, abspath)); [error] = await safe(fs.promises.symlink(header.linkname, abspath));
} else { } else {
debug(`tarExtract: ignoring unknown entry: ${header.name} ${header.type}`); log(`tarExtract: ignoring unknown entry: ${header.name} ${header.type}`);
entryStream.resume(); // drain entryStream.resume(); // drain
} }
@@ -220,7 +220,7 @@ async function tarExtract(inStream, dataLayout, encryption, progressCallback) {
[error] = await safe(fs.promises.lutimes(abspath, now /* atime */, header.mtime)); // for dirs, mtime will get overwritten [error] = await safe(fs.promises.lutimes(abspath, now /* atime */, header.mtime)); // for dirs, mtime will get overwritten
next(error); next(error);
}); });
extract.on('finish', () => debug(`tarExtract: extracted ${entryCount} entries`)); extract.on('finish', () => log(`tarExtract: extracted ${entryCount} entries`));
const gunzip = zlib.createGunzip({}); const gunzip = zlib.createGunzip({});
const ps = new ProgressStream({ interval: 10000 }); const ps = new ProgressStream({ interval: 10000 });
@@ -242,7 +242,7 @@ async function tarExtract(inStream, dataLayout, encryption, progressCallback) {
if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, `tarExtract pipeline error: ${error.message}`); if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, `tarExtract pipeline error: ${error.message}`);
} }
debug(`tarExtract: pipeline finished: ${JSON.stringify(ps.stats())}`); log(`tarExtract: pipeline finished: ${JSON.stringify(ps.stats())}`);
} }
async function download(backupSite, remotePath, dataLayout, progressCallback) { async function download(backupSite, remotePath, dataLayout, progressCallback) {
@@ -251,9 +251,9 @@ async function download(backupSite, remotePath, dataLayout, progressCallback) {
assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout'); assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
assert.strictEqual(typeof progressCallback, 'function'); assert.strictEqual(typeof progressCallback, 'function');
debug(`download: Downloading ${remotePath} to ${dataLayout.toString()}`); log(`download: Downloading ${remotePath} to ${dataLayout.toString()}`);
await promiseRetry({ times: 3, interval: 20000, debug }, async () => { await promiseRetry({ times: 3, interval: 20000, debug: log }, async () => {
progressCallback({ message: `Downloading backup ${remotePath}` }); progressCallback({ message: `Downloading backup ${remotePath}` });
const sourceStream = await backupSites.storageApi(backupSite).download(backupSite.config, remotePath); const sourceStream = await backupSites.storageApi(backupSite).download(backupSite.config, remotePath);
@@ -267,9 +267,9 @@ async function upload(backupSite, remotePath, dataLayout, progressCallback) {
assert.strictEqual(typeof dataLayout, 'object'); assert.strictEqual(typeof dataLayout, 'object');
assert.strictEqual(typeof progressCallback, 'function'); assert.strictEqual(typeof progressCallback, 'function');
debug(`upload: uploading to site ${backupSite.id} path ${remotePath} (encrypted: ${!!backupSite.encryption}) dataLayout ${dataLayout.toString()}`); log(`upload: uploading to site ${backupSite.id} path ${remotePath} (encrypted: ${!!backupSite.encryption}) dataLayout ${dataLayout.toString()}`);
return await promiseRetry({ times: 5, interval: 20000, debug }, async () => { return await promiseRetry({ times: 5, interval: 20000, debug: log }, async () => {
progressCallback({ message: `Uploading backup ${remotePath}` }); progressCallback({ message: `Uploading backup ${remotePath}` });
const uploader = await backupSites.storageApi(backupSite).upload(backupSite.config, backupSite.limits, remotePath); const uploader = await backupSites.storageApi(backupSite).upload(backupSite.config, backupSite.limits, remotePath);
@@ -296,7 +296,7 @@ async function verify(backupSite, remotePath, integrityMap, progressCallback) {
assert(util.types.isMap(integrityMap), 'integrityMap should be a Map'); assert(util.types.isMap(integrityMap), 'integrityMap should be a Map');
assert.strictEqual(typeof progressCallback, 'function'); assert.strictEqual(typeof progressCallback, 'function');
debug(`verify: Verifying ${remotePath}`); log(`verify: Verifying ${remotePath}`);
const inStream = await backupSites.storageApi(backupSite).download(backupSite.config, remotePath); const inStream = await backupSites.storageApi(backupSite).download(backupSite.config, remotePath);
@@ -305,17 +305,17 @@ async function verify(backupSite, remotePath, integrityMap, progressCallback) {
const extract = tar.extract(); const extract = tar.extract();
extract.on('entry', async function (header, entryStream, next) { extract.on('entry', async function (header, entryStream, next) {
if (path.isAbsolute(header.name)) { if (path.isAbsolute(header.name)) {
debug(`verify: ignoring absolute path ${header.name}`); log(`verify: ignoring absolute path ${header.name}`);
return next(); return next();
} }
debug(`verify: ${header.name} ${header.size} ${header.type}`); log(`verify: ${header.name} ${header.size} ${header.type}`);
if (header.type === 'file') { if (header.type === 'file') {
++fileCount; ++fileCount;
} }
entryStream.resume(); // drain entryStream.resume(); // drain
next(); next();
}); });
extract.on('finish', () => debug('verify: extract finished')); extract.on('finish', () => log('verify: extract finished'));
const hash = new HashStream(); const hash = new HashStream();
const gunzip = zlib.createGunzip({}); const gunzip = zlib.createGunzip({});
@@ -336,7 +336,7 @@ async function verify(backupSite, remotePath, integrityMap, progressCallback) {
} }
const integrity = integrityMap.get('.'); const integrity = integrityMap.get('.');
debug(`verify: Expecting: ${JSON.stringify(integrity)} Actual: size:${ps.stats().transferred} filecount:${fileCount} digest:${hash.digest()}`); log(`verify: Expecting: ${JSON.stringify(integrity)} Actual: size:${ps.stats().transferred} filecount:${fileCount} digest:${hash.digest()}`);
const messages = []; const messages = [];
if (integrity.size !== ps.stats().transferred) messages.push(`Size mismatch. Expected: ${integrity.size} Actual: ${ps.stats().transferred}`); if (integrity.size !== ps.stats().transferred) messages.push(`Size mismatch. Expected: ${integrity.size} Actual: ${ps.stats().transferred}`);

6
src/backupintegrity.js
View File

@@ -5,10 +5,10 @@ import backupSites from './backupsites.js';
import BoxError from './boxerror.js'; import BoxError from './boxerror.js';
import consumers from 'node:stream/consumers'; import consumers from 'node:stream/consumers';
import crypto from 'node:crypto'; import crypto from 'node:crypto';
import debugModule from 'debug'; import logger from './logger.js';
import safe from 'safetydance'; import safe from 'safetydance';
const debug = debugModule('box:backupintegrity'); const { log, trace } = logger('backupintegrity');
async function downloadBackupInfo(backupSite, backup) { async function downloadBackupInfo(backupSite, backup) {
@@ -47,7 +47,7 @@ async function verify(backup, backupSite, progressCallback) {
if (verifyError) messages.push(`Failed to verify ${backup.remotePath}: ${verifyError.message}`); if (verifyError) messages.push(`Failed to verify ${backup.remotePath}: ${verifyError.message}`);
if (verifyMessages) messages.push(...verifyMessages); if (verifyMessages) messages.push(...verifyMessages);
debug(`verified: ${backup.remotePath} ${JSON.stringify(messages, null, 4)}`); log(`verified: ${backup.remotePath} ${JSON.stringify(messages, null, 4)}`);
stats.duration = Date.now() - stats.startTime; stats.duration = Date.now() - stats.startTime;
return { stats, messages: messages.slice(0, 50) }; // keep rsync fails to 50 to not overflow db return { stats, messages: messages.slice(0, 50) }; // keep rsync fails to 50 to not overflow db

8
src/backups.js
View File

@@ -1,13 +1,13 @@
import assert from 'node:assert'; import assert from 'node:assert';
import BoxError from './boxerror.js'; import BoxError from './boxerror.js';
import database from './database.js'; import database from './database.js';
import debugModule from 'debug'; import logger from './logger.js';
import eventlog from './eventlog.js'; import eventlog from './eventlog.js';
import hat from './hat.js'; import hat from './hat.js';
import safe from 'safetydance'; import safe from 'safetydance';
import tasks from './tasks.js'; import tasks from './tasks.js';
const debug = debugModule('box:backups'); const { log, trace } = logger('backups');
const BACKUP_TYPE_APP = 'app'; const BACKUP_TYPE_APP = 'app';
const BACKUP_STATE_NORMAL = 'normal'; const BACKUP_STATE_NORMAL = 'normal';
@@ -222,11 +222,11 @@ async function startIntegrityCheck(backup, auditSource) {
// background // background
tasks.startTask(taskId, {}) tasks.startTask(taskId, {})
.then(async (status) => { .then(async (status) => {
debug(`startIntegrityCheck: task completed`); log(`startIntegrityCheck: task completed`);
await eventlog.add(eventlog.ACTION_BACKUP_INTEGRITY_FINISH, auditSource, { status, taskId, backupId: backup.id }); await eventlog.add(eventlog.ACTION_BACKUP_INTEGRITY_FINISH, auditSource, { status, taskId, backupId: backup.id });
}) })
.catch(async (error) => { .catch(async (error) => {
debug(`startIntegrityCheck: task error. ${error.message}`); log(`startIntegrityCheck: task error. ${error.message}`);
await eventlog.add(eventlog.ACTION_BACKUP_INTEGRITY_FINISH, auditSource, { errorMessage: error.message, taskId, backupId: backup.id }); await eventlog.add(eventlog.ACTION_BACKUP_INTEGRITY_FINISH, auditSource, { errorMessage: error.message, taskId, backupId: backup.id });
}); });

20
src/backupsites.js
View File

@@ -6,7 +6,7 @@ import cron from './cron.js';
import { CronTime } from 'cron'; import { CronTime } from 'cron';
import crypto from 'node:crypto'; import crypto from 'node:crypto';
import database from './database.js'; import database from './database.js';
import debugModule from 'debug'; import logger from './logger.js';
import eventlog from './eventlog.js'; import eventlog from './eventlog.js';
import hush from './hush.js'; import hush from './hush.js';
import locks from './locks.js'; import locks from './locks.js';
@@ -18,7 +18,7 @@ import storageFilesystem from './storage/filesystem.js';
import storageS3 from './storage/s3.js'; import storageS3 from './storage/s3.js';
import storageGcs from './storage/gcs.js'; import storageGcs from './storage/gcs.js';
const debug = debugModule('box:backups'); const { log, trace } = logger('backups');
// format: rsync or tgz // format: rsync or tgz
@@ -304,7 +304,7 @@ async function del(backupSite, auditSource) {
assert.strictEqual(typeof backupSite, 'object'); assert.strictEqual(typeof backupSite, 'object');
assert.strictEqual(typeof auditSource, 'object'); assert.strictEqual(typeof auditSource, 'object');
await safe(storageApi(backupSite).teardown(backupSite.config), { debug }); // ignore error await safe(storageApi(backupSite).teardown(backupSite.config), { debug: log }); // ignore error
const queries = [ const queries = [
{ query: 'DELETE FROM archives WHERE backupId IN (SELECT id FROM backups WHERE siteId=?)', args: [ backupSite.id ] }, { query: 'DELETE FROM archives WHERE backupId IN (SELECT id FROM backups WHERE siteId=?)', args: [ backupSite.id ] },
@@ -437,12 +437,12 @@ async function setConfig(backupSite, newConfig, auditSource) {
newConfig = structuredClone(newConfig); // make a copy newConfig = structuredClone(newConfig); // make a copy
storageApi(backupSite).injectPrivateFields(newConfig, oldConfig); storageApi(backupSite).injectPrivateFields(newConfig, oldConfig);
debug('setConfig: validating new storage configuration'); log('setConfig: validating new storage configuration');
const sanitizedConfig = await storageApi(backupSite).verifyConfig({ id: backupSite.id, provider: backupSite.provider, config: newConfig }); const sanitizedConfig = await storageApi(backupSite).verifyConfig({ id: backupSite.id, provider: backupSite.provider, config: newConfig });
await update(backupSite, { config: sanitizedConfig }); await update(backupSite, { config: sanitizedConfig });
debug('setConfig: setting up new storage configuration'); log('setConfig: setting up new storage configuration');
await storageApi(backupSite).setup(sanitizedConfig); await storageApi(backupSite).setup(sanitizedConfig);
await eventlog.add(eventlog.ACTION_BACKUP_SITE_UPDATE, auditSource, { name: backupSite.name, config: storageApi(backupSite).removePrivateFields(newConfig) }); await eventlog.add(eventlog.ACTION_BACKUP_SITE_UPDATE, auditSource, { name: backupSite.name, config: storageApi(backupSite).removePrivateFields(newConfig) });
@@ -487,13 +487,13 @@ async function add(data, auditSource) {
const id = crypto.randomUUID(); const id = crypto.randomUUID();
if (!safe.fs.mkdirSync(`${paths.BACKUP_INFO_DIR}/${id}`)) throw new BoxError(BoxError.FS_ERROR, `Failed to create info dir: ${safe.error.message}`); if (!safe.fs.mkdirSync(`${paths.BACKUP_INFO_DIR}/${id}`)) throw new BoxError(BoxError.FS_ERROR, `Failed to create info dir: ${safe.error.message}`);
debug('add: validating new storage configuration'); log('add: validating new storage configuration');
const sanitizedConfig = await storageApi({ provider }).verifyConfig({id, provider, config }); const sanitizedConfig = await storageApi({ provider }).verifyConfig({id, provider, config });
await database.query('INSERT INTO backupSites (id, name, provider, configJson, contentsJson, limitsJson, integrityKeyPairJson, retentionJson, schedule, encryptionJson, format, enableForUpdates) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', await database.query('INSERT INTO backupSites (id, name, provider, configJson, contentsJson, limitsJson, integrityKeyPairJson, retentionJson, schedule, encryptionJson, format, enableForUpdates) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
[ id, name, provider, JSON.stringify(sanitizedConfig), JSON.stringify(contents), JSON.stringify(limits), JSON.stringify(integrityKeyPair), JSON.stringify(retention), schedule, JSON.stringify(encryption), format, enableForUpdates ]); [ id, name, provider, JSON.stringify(sanitizedConfig), JSON.stringify(contents), JSON.stringify(limits), JSON.stringify(integrityKeyPair), JSON.stringify(retention), schedule, JSON.stringify(encryption), format, enableForUpdates ]);
debug('add: setting up new storage configuration'); log('add: setting up new storage configuration');
await storageApi({ provider }).setup(sanitizedConfig); await storageApi({ provider }).setup(sanitizedConfig);
await eventlog.add(eventlog.ACTION_BACKUP_SITE_ADD, auditSource, { id, name, provider, contents, schedule, format }); await eventlog.add(eventlog.ACTION_BACKUP_SITE_ADD, auditSource, { id, name, provider, contents, schedule, format });
@@ -504,7 +504,7 @@ async function add(data, auditSource) {
async function addDefault(auditSource) { async function addDefault(auditSource) {
assert.strictEqual(typeof auditSource, 'object'); assert.strictEqual(typeof auditSource, 'object');
debug('addDefault: adding default backup site'); log('addDefault: adding default backup site');
const defaultBackupSite = { const defaultBackupSite = {
name: 'Default', name: 'Default',
provider: 'filesystem', provider: 'filesystem',
@@ -536,7 +536,7 @@ async function createPseudo(data) {
encryption.encryptionPasswordHint = ''; encryption.encryptionPasswordHint = '';
} }
debug('add: validating new storage configuration'); log('add: validating new storage configuration');
const sanitizedConfig = await storageApi({ provider }).verifyConfig({id, provider, config }); const sanitizedConfig = await storageApi({ provider }).verifyConfig({id, provider, config });
return { id, format, provider, config: sanitizedConfig, encryption }; return { id, format, provider, config: sanitizedConfig, encryption };
} }
@@ -547,7 +547,7 @@ async function reinitAll() {
if (!safe.fs.mkdirSync(`${paths.BACKUP_INFO_DIR}/${site.id}`, { recursive: true })) throw new BoxError(BoxError.FS_ERROR, `Failed to create info dir: ${safe.error.message}`); if (!safe.fs.mkdirSync(`${paths.BACKUP_INFO_DIR}/${site.id}`, { recursive: true })) throw new BoxError(BoxError.FS_ERROR, `Failed to create info dir: ${safe.error.message}`);
const status = await getStatus(site); const status = await getStatus(site);
if (status.state === 'active') continue; if (status.state === 'active') continue;
safe(remount(site), { debug }); // background safe(remount(site), { debug: log }); // background
} }
} }

64
src/backuptask.js
View File

@@ -8,7 +8,7 @@ import constants from './constants.js';
import crypto from 'node:crypto'; import crypto from 'node:crypto';
import DataLayout from './datalayout.js'; import DataLayout from './datalayout.js';
import database from './database.js'; import database from './database.js';
import debugModule from 'debug'; import logger from './logger.js';
import df from './df.js'; import df from './df.js';
import locks from './locks.js'; import locks from './locks.js';
import path from 'node:path'; import path from 'node:path';
@@ -20,7 +20,7 @@ import shellModule from './shell.js';
import stream from 'stream/promises'; import stream from 'stream/promises';
import util from 'util'; import util from 'util';
const debug = debugModule('box:backuptask'); const { log, trace } = logger('backuptask');
const shell = shellModule('backuptask'); const shell = shellModule('backuptask');
@@ -40,22 +40,22 @@ async function checkPreconditions(backupSite, dataLayout) {
// check mount status before uploading // check mount status before uploading
const status = await backupSites.ensureMounted(backupSite); const status = await backupSites.ensureMounted(backupSite);
debug(`checkPreconditions: mount point status is ${JSON.stringify(status)}`); log(`checkPreconditions: mount point status is ${JSON.stringify(status)}`);
if (status.state !== 'active') throw new BoxError(BoxError.MOUNT_ERROR, `Backup endpoint is not active: ${status.message}`); if (status.state !== 'active') throw new BoxError(BoxError.MOUNT_ERROR, `Backup endpoint is not active: ${status.message}`);
// check availabe size. this requires root for df to work // check availabe size. this requires root for df to work
const available = await backupSites.storageApi(backupSite).getAvailableSize(backupSite.config); const available = await backupSites.storageApi(backupSite).getAvailableSize(backupSite.config);
let used = 0; let used = 0;
for (const localPath of dataLayout.localPaths()) { for (const localPath of dataLayout.localPaths()) {
debug(`checkPreconditions: getting disk usage of ${localPath}`); log(`checkPreconditions: getting disk usage of ${localPath}`);
// du can error when files go missing as it is computing the size. it still prints some size anyway // du can error when files go missing as it is computing the size. it still prints some size anyway
// to match df output in getAvailableSize() we must use disk usage size here and not apparent size // to match df output in getAvailableSize() we must use disk usage size here and not apparent size
const [duError, result] = await safe(shell.spawn('du', [ '--dereference-args', '--summarize', '--block-size=1', '--exclude=*.lock', '--exclude=dovecot.list.index.log.*', localPath], { encoding: 'utf8' })); const [duError, result] = await safe(shell.spawn('du', [ '--dereference-args', '--summarize', '--block-size=1', '--exclude=*.lock', '--exclude=dovecot.list.index.log.*', localPath], { encoding: 'utf8' }));
if (duError) debug(`checkPreconditions: du error for ${localPath}. code: ${duError.code} stderror: ${duError.stderr}`); if (duError) log(`checkPreconditions: du error for ${localPath}. code: ${duError.code} stderror: ${duError.stderr}`);
used += parseInt(duError ? duError.stdout : result, 10); used += parseInt(duError ? duError.stdout : result, 10);
} }
debug(`checkPreconditions: total required=${used} available=${available}`); log(`checkPreconditions: total required=${used} available=${available}`);
const needed = 0.6 * used + (1024 * 1024 * 1024); // check if there is atleast 1GB left afterwards. aim for 60% because rsync/tgz won't need full 100% const needed = 0.6 * used + (1024 * 1024 * 1024); // check if there is atleast 1GB left afterwards. aim for 60% because rsync/tgz won't need full 100%
if (available <= needed) throw new BoxError(BoxError.FS_ERROR, `Not enough disk space for backup. Needed: ${df.prettyBytes(needed)} Available: ${df.prettyBytes(available)}`); if (available <= needed) throw new BoxError(BoxError.FS_ERROR, `Not enough disk space for backup. Needed: ${df.prettyBytes(needed)} Available: ${df.prettyBytes(available)}`);
@@ -81,7 +81,7 @@ async function upload(remotePath, siteId, dataLayoutString, progressCallback) {
assert.strictEqual(typeof dataLayoutString, 'string'); assert.strictEqual(typeof dataLayoutString, 'string');
assert.strictEqual(typeof progressCallback, 'function'); assert.strictEqual(typeof progressCallback, 'function');
debug(`upload: path ${remotePath} site ${siteId} dataLayout ${dataLayoutString}`); log(`upload: path ${remotePath} site ${siteId} dataLayout ${dataLayoutString}`);
const backupSite = await backupSites.get(siteId); const backupSite = await backupSites.get(siteId);
if (!backupSite) throw new BoxError(BoxError.NOT_FOUND, 'Backup site not found'); if (!backupSite) throw new BoxError(BoxError.NOT_FOUND, 'Backup site not found');
@@ -99,7 +99,7 @@ async function upload(remotePath, siteId, dataLayoutString, progressCallback) {
// - rsync: size (final backup size) will be different from what was transferred (only changed files) // - rsync: size (final backup size) will be different from what was transferred (only changed files)
// stats.fileCount and stats.size are stored in db and should match up what is written into .backupinfo // stats.fileCount and stats.size are stored in db and should match up what is written into .backupinfo
const { stats, integrityMap } = await backupFormats.api(backupSite.format).upload(backupSite, remotePath, dataLayout, progressCallback); const { stats, integrityMap } = await backupFormats.api(backupSite.format).upload(backupSite, remotePath, dataLayout, progressCallback);
debug(`upload: path ${remotePath} site ${siteId} uploaded: ${JSON.stringify(stats)}`); log(`upload: path ${remotePath} site ${siteId} uploaded: ${JSON.stringify(stats)}`);
progressCallback({ message: `Uploading integrity information to ${remotePath}.backupinfo` }); progressCallback({ message: `Uploading integrity information to ${remotePath}.backupinfo` });
const signature = await uploadBackupInfo(backupSite, remotePath, integrityMap); const signature = await uploadBackupInfo(backupSite, remotePath, integrityMap);
@@ -112,7 +112,7 @@ async function download(backupSite, remotePath, dataLayout, progressCallback) {
assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout'); assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
assert.strictEqual(typeof progressCallback, 'function'); assert.strictEqual(typeof progressCallback, 'function');
debug(`download: Downloading ${remotePath} of format ${backupSite.format} (encrypted: ${!!backupSite.encryption}) to ${dataLayout.toString()}`); log(`download: Downloading ${remotePath} of format ${backupSite.format} (encrypted: ${!!backupSite.encryption}) to ${dataLayout.toString()}`);
await backupFormats.api(backupSite.format).download(backupSite, remotePath, dataLayout, progressCallback); await backupFormats.api(backupSite.format).download(backupSite, remotePath, dataLayout, progressCallback);
} }
@@ -128,10 +128,10 @@ async function restore(backupSite, remotePath, progressCallback) {
await download(backupSite, remotePath, dataLayout, progressCallback); await download(backupSite, remotePath, dataLayout, progressCallback);
debug('restore: download completed, importing database'); log('restore: download completed, importing database');
await database.importFromFile(`${dataLayout.localRoot()}/box.mysqldump`); await database.importFromFile(`${dataLayout.localRoot()}/box.mysqldump`);
debug('restore: database imported'); log('restore: database imported');
await locks.releaseAll(); // clear the locks table in database await locks.releaseAll(); // clear the locks table in database
} }
@@ -155,7 +155,7 @@ async function downloadApp(app, restoreConfig, progressCallback) {
} }
await download(backupSite, remotePath, dataLayout, progressCallback); await download(backupSite, remotePath, dataLayout, progressCallback);
debug('downloadApp: time: %s', (new Date() - startTime)/1000); log('downloadApp: time: %s', (new Date() - startTime)/1000);
} }
async function runBackupUpload(uploadConfig, progressCallback) { async function runBackupUpload(uploadConfig, progressCallback) {
@@ -172,21 +172,21 @@ async function runBackupUpload(uploadConfig, progressCallback) {
const envCopy = Object.assign({}, process.env); const envCopy = Object.assign({}, process.env);
if (backupSite.limits?.memoryLimit >= 2*1024*1024*1024) { if (backupSite.limits?.memoryLimit >= 2*1024*1024*1024) {
const heapSize = Math.min((backupSite.limits.memoryLimit/1024/1024) - 256, 8192); const heapSize = Math.min((backupSite.limits.memoryLimit/1024/1024) - 256, 8192);
debug(`runBackupUpload: adjusting heap size to ${heapSize}M`); log(`runBackupUpload: adjusting heap size to ${heapSize}M`);
envCopy.NODE_OPTIONS = `--max-old-space-size=${heapSize}`; envCopy.NODE_OPTIONS = `--max-old-space-size=${heapSize}`;
} }
let lastMessage = null; // the script communicates error result as a string let lastMessage = null; // the script communicates error result as a string
function onMessage(progress) { // this is { message } or { result } function onMessage(progress) { // this is { message } or { result }
if ('message' in progress) return progressCallback({ message: `${progress.message} (${progressTag})` }); if ('message' in progress) return progressCallback({ message: `${progress.message} (${progressTag})` });
debug(`runBackupUpload: result - ${JSON.stringify(progress)}`); log(`runBackupUpload: result - ${JSON.stringify(progress)}`);
lastMessage = progress; lastMessage = progress;
} }
// do not use debug for logging child output because it already has timestamps via it's own debug // do not use debug for logging child output because it already has timestamps via it's own debug
const [error] = await safe(shell.sudo([ BACKUP_UPLOAD_CMD, remotePath, backupSite.id, dataLayout.toString() ], { env: envCopy, preserveEnv: true, onMessage, logger: process.stdout.write })); const [error] = await safe(shell.sudo([ BACKUP_UPLOAD_CMD, remotePath, backupSite.id, dataLayout.toString() ], { env: envCopy, preserveEnv: true, onMessage, logger: process.stdout.write }));
if (error && (error.code === null /* signal */ || (error.code !== 0 && error.code !== 50))) { // backuptask crashed if (error && (error.code === null /* signal */ || (error.code !== 0 && error.code !== 50))) { // backuptask crashed
debug(`runBackupUpload: backuptask crashed`, error); log(`runBackupUpload: backuptask crashed`, error);
throw new BoxError(BoxError.INTERNAL_ERROR, 'Backuptask crashed'); throw new BoxError(BoxError.INTERNAL_ERROR, 'Backuptask crashed');
} else if (error && error.code === 50) { // exited with error } else if (error && error.code === 50) { // exited with error
throw new BoxError(BoxError.EXTERNAL_ERROR, lastMessage.errorMessage); throw new BoxError(BoxError.EXTERNAL_ERROR, lastMessage.errorMessage);
@@ -203,7 +203,7 @@ async function snapshotBox(progressCallback) {
const startTime = new Date(); const startTime = new Date();
await database.exportToFile(`${paths.BOX_DATA_DIR}/box.mysqldump`); await database.exportToFile(`${paths.BOX_DATA_DIR}/box.mysqldump`);
debug(`snapshotBox: took ${(new Date() - startTime)/1000} seconds`); log(`snapshotBox: took ${(new Date() - startTime)/1000} seconds`);
} }
async function uploadBoxSnapshot(backupSite, progressCallback) { async function uploadBoxSnapshot(backupSite, progressCallback) {
@@ -230,7 +230,7 @@ async function uploadBoxSnapshot(backupSite, progressCallback) {
const { stats, integrity } = await runBackupUpload(uploadConfig, progressCallback); const { stats, integrity } = await runBackupUpload(uploadConfig, progressCallback);
debug(`uploadBoxSnapshot: took ${(new Date() - startTime)/1000} seconds`); log(`uploadBoxSnapshot: took ${(new Date() - startTime)/1000} seconds`);
await backupSites.setSnapshotInfo(backupSite, 'box', { timestamp: new Date().toISOString() }); await backupSites.setSnapshotInfo(backupSite, 'box', { timestamp: new Date().toISOString() });
@@ -246,17 +246,17 @@ async function copy(backupSite, srcRemotePath, destRemotePath, progressCallback)
const startTime = new Date(); const startTime = new Date();
const [copyError] = await safe(backupFormats.api(backupSite.format).copy(backupSite, srcRemotePath, destRemotePath, progressCallback)); const [copyError] = await safe(backupFormats.api(backupSite.format).copy(backupSite, srcRemotePath, destRemotePath, progressCallback));
if (copyError) { if (copyError) {
debug(`copy: copy to ${destRemotePath} errored. error: ${copyError.message}`); log(`copy: copy to ${destRemotePath} errored. error: ${copyError.message}`);
throw copyError; throw copyError;
} }
debug(`copy: copied successfully to ${destRemotePath}. Took ${(new Date() - startTime)/1000} seconds`); log(`copy: copied successfully to ${destRemotePath}. Took ${(new Date() - startTime)/1000} seconds`);
const [copyChecksumError] = await safe(backupSites.storageApi(backupSite).copy(backupSite.config, `${srcRemotePath}.backupinfo`, `${destRemotePath}.backupinfo`, progressCallback)); const [copyChecksumError] = await safe(backupSites.storageApi(backupSite).copy(backupSite.config, `${srcRemotePath}.backupinfo`, `${destRemotePath}.backupinfo`, progressCallback));
if (copyChecksumError) { if (copyChecksumError) {
debug(`copy: copy to ${destRemotePath} errored. error: ${copyChecksumError.message}`); log(`copy: copy to ${destRemotePath} errored. error: ${copyChecksumError.message}`);
throw copyChecksumError; throw copyChecksumError;
} }
debug(`copy: copied backupinfo successfully to ${destRemotePath}.backupinfo`); log(`copy: copied backupinfo successfully to ${destRemotePath}.backupinfo`);
} }
async function backupBox(backupSite, appBackupsMap, tag, options, progressCallback) { async function backupBox(backupSite, appBackupsMap, tag, options, progressCallback) {
@@ -281,7 +281,7 @@ async function backupBox(backupSite, appBackupsMap, tag, options, progressCallba
duration: acc.duration + cur.upload.duration, duration: acc.duration + cur.upload.duration,
}), stats.upload); }), stats.upload);
debug(`backupBox: rotating box snapshot of ${backupSite.id} to id ${remotePath}. ${JSON.stringify(stats)}`); log(`backupBox: rotating box snapshot of ${backupSite.id} to id ${remotePath}. ${JSON.stringify(stats)}`);
const data = { const data = {
remotePath, remotePath,
@@ -329,7 +329,7 @@ async function snapshotApp(app, progressCallback) {
await services.runBackupCommand(app); await services.runBackupCommand(app);
await services.backupAddons(app, app.manifest.addons); await services.backupAddons(app, app.manifest.addons);
debug(`snapshotApp: ${app.fqdn} took ${(new Date() - startTime)/1000} seconds`); log(`snapshotApp: ${app.fqdn} took ${(new Date() - startTime)/1000} seconds`);
} }
async function uploadAppSnapshot(backupSite, app, progressCallback) { async function uploadAppSnapshot(backupSite, app, progressCallback) {
@@ -358,7 +358,7 @@ async function uploadAppSnapshot(backupSite, app, progressCallback) {
const { stats, integrity } = await runBackupUpload(uploadConfig, progressCallback); const { stats, integrity } = await runBackupUpload(uploadConfig, progressCallback);
debug(`uploadAppSnapshot: ${app.fqdn} uploaded to ${remotePath}. ${(new Date() - startTime)/1000} seconds`); log(`uploadAppSnapshot: ${app.fqdn} uploaded to ${remotePath}. ${(new Date() - startTime)/1000} seconds`);
await backupSites.setSnapshotInfo(backupSite, app.id, { timestamp: new Date().toISOString(), manifest: app.manifest }); await backupSites.setSnapshotInfo(backupSite, app.id, { timestamp: new Date().toISOString(), manifest: app.manifest });
@@ -385,7 +385,7 @@ async function backupAppWithTag(app, backupSite, tag, options, progressCallback)
const manifest = app.manifest; const manifest = app.manifest;
const remotePath = addFileExtension(backupSite, `${tag}/app_${app.fqdn}_v${manifest.version}`); const remotePath = addFileExtension(backupSite, `${tag}/app_${app.fqdn}_v${manifest.version}`);
debug(`backupAppWithTag: rotating ${app.fqdn} snapshot of ${backupSite.id} to path ${remotePath}`); log(`backupAppWithTag: rotating ${app.fqdn} snapshot of ${backupSite.id} to path ${remotePath}`);
const data = { const data = {
remotePath, remotePath,
@@ -456,7 +456,7 @@ async function uploadMailSnapshot(backupSite, progressCallback) {
const { stats, integrity } = await runBackupUpload(uploadConfig, progressCallback); const { stats, integrity } = await runBackupUpload(uploadConfig, progressCallback);
debug(`uploadMailSnapshot: took ${(new Date() - startTime)/1000} seconds`); log(`uploadMailSnapshot: took ${(new Date() - startTime)/1000} seconds`);
await backupSites.setSnapshotInfo(backupSite, 'mail', { timestamp: new Date().toISOString() }); await backupSites.setSnapshotInfo(backupSite, 'mail', { timestamp: new Date().toISOString() });
@@ -469,7 +469,7 @@ async function backupMailWithTag(backupSite, tag, options, progressCallback) {
assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof options, 'object');
assert.strictEqual(typeof progressCallback, 'function'); assert.strictEqual(typeof progressCallback, 'function');
debug(`backupMailWithTag: backing up mail with tag ${tag}`); log(`backupMailWithTag: backing up mail with tag ${tag}`);
const uploadStartTime = Date.now(); const uploadStartTime = Date.now();
const uploadResult = await uploadMailSnapshot(backupSite, progressCallback); // { stats, integrity } const uploadResult = await uploadMailSnapshot(backupSite, progressCallback); // { stats, integrity }
@@ -477,7 +477,7 @@ async function backupMailWithTag(backupSite, tag, options, progressCallback) {
const remotePath = addFileExtension(backupSite, `${tag}/mail_v${constants.VERSION}`); const remotePath = addFileExtension(backupSite, `${tag}/mail_v${constants.VERSION}`);
debug(`backupMailWithTag: rotating mail snapshot of ${backupSite.id} to ${remotePath}`); log(`backupMailWithTag: rotating mail snapshot of ${backupSite.id} to ${remotePath}`);
const data = { const data = {
remotePath, remotePath,
@@ -519,7 +519,7 @@ async function downloadMail(backupSite, remotePath, progressCallback) {
const startTime = new Date(); const startTime = new Date();
await download(backupSite, remotePath, dataLayout, progressCallback); await download(backupSite, remotePath, dataLayout, progressCallback);
debug('downloadMail: time: %s', (new Date() - startTime)/1000); log('downloadMail: time: %s', (new Date() - startTime)/1000);
} }
// this function is called from external process. calling process is expected to have a lock // this function is called from external process. calling process is expected to have a lock
@@ -544,11 +544,11 @@ async function fullBackup(backupSiteId, options, progressCallback) {
percent += step; percent += step;
if (!app.enableBackup) { if (!app.enableBackup) {
debug(`fullBackup: skipped backup ${app.fqdn} (${i+1}/${allApps.length}) since automatic backup disabled`); log(`fullBackup: skipped backup ${app.fqdn} (${i+1}/${allApps.length}) since automatic backup disabled`);
continue; // nothing to backup continue; // nothing to backup
} }
if (!backupSites.hasContent(backupSite, app.id)) { if (!backupSites.hasContent(backupSite, app.id)) {
debug(`fullBackup: skipped backup ${app.fqdn} (${i+1}/${allApps.length}) as it is not in site contents`); log(`fullBackup: skipped backup ${app.fqdn} (${i+1}/${allApps.length}) as it is not in site contents`);
continue; continue;
} }
@@ -556,7 +556,7 @@ async function fullBackup(backupSiteId, options, progressCallback) {
await locks.wait(`${locks.TYPE_APP_BACKUP_PREFIX}${app.id}`); await locks.wait(`${locks.TYPE_APP_BACKUP_PREFIX}${app.id}`);
const startTime = new Date(); const startTime = new Date();
const [appBackupError, appBackupResult] = await safe(backupAppWithTag(app, backupSite, tag, options, (progress) => progressCallback({ percent, message: progress.message }))); const [appBackupError, appBackupResult] = await safe(backupAppWithTag(app, backupSite, tag, options, (progress) => progressCallback({ percent, message: progress.message })));
debug(`fullBackup: app ${app.fqdn} backup finished. Took ${(new Date() - startTime)/1000} seconds`); log(`fullBackup: app ${app.fqdn} backup finished. Took ${(new Date() - startTime)/1000} seconds`);
await locks.release(`${locks.TYPE_APP_BACKUP_PREFIX}${app.id}`); await locks.release(`${locks.TYPE_APP_BACKUP_PREFIX}${app.id}`);
if (appBackupError) throw appBackupError; if (appBackupError) throw appBackupError;
if (appBackupResult) appBackupsMap.set(appBackupResult.id, appBackupResult.stats); // backupId can be null if in BAD_STATE and never backed up if (appBackupResult) appBackupsMap.set(appBackupResult.id, appBackupResult.stats); // backupId can be null if in BAD_STATE and never backed up

6
src/branding.js
View File

@@ -2,13 +2,13 @@ import apps from './apps.js';
import assert from 'node:assert'; import assert from 'node:assert';
import BoxError from './boxerror.js'; import BoxError from './boxerror.js';
import constants from './constants.js'; import constants from './constants.js';
import debugModule from 'debug'; import logger from './logger.js';
import eventlog from './eventlog.js'; import eventlog from './eventlog.js';
import paths from './paths.js'; import paths from './paths.js';
import safe from 'safetydance'; import safe from 'safetydance';
import settings from './settings.js'; import settings from './settings.js';
const debug = debugModule('box:branding'); const { log, trace } = logger('branding');
async function getCloudronName() { async function getCloudronName() {
@@ -28,7 +28,7 @@ async function setCloudronName(name, auditSource) {
// mark apps using oidc addon to be reconfigured // mark apps using oidc addon to be reconfigured
const [, installedApps] = await safe(apps.list()); const [, installedApps] = await safe(apps.list());
await safe(apps.configureApps(installedApps.filter((a) => !!a.manifest.addons?.oidc), { scheduleNow: true }, auditSource), { debug }); await safe(apps.configureApps(installedApps.filter((a) => !!a.manifest.addons?.oidc), { scheduleNow: true }, auditSource), { debug: log });
await settings.set(settings.CLOUDRON_NAME_KEY, name); await settings.set(settings.CLOUDRON_NAME_KEY, name);
await eventlog.add(eventlog.ACTION_BRANDING_NAME, auditSource, { name }); await eventlog.add(eventlog.ACTION_BRANDING_NAME, auditSource, { name });

8
src/community.js
View File

@@ -1,12 +1,12 @@
import assert from 'node:assert'; import assert from 'node:assert';
import BoxError from './boxerror.js'; import BoxError from './boxerror.js';
import debugModule from 'debug'; import logger from './logger.js';
import manifestFormat from '@cloudron/manifest-format'; import manifestFormat from '@cloudron/manifest-format';
import promiseRetry from './promise-retry.js'; import promiseRetry from './promise-retry.js';
import safe from 'safetydance'; import safe from 'safetydance';
import superagent from '@cloudron/superagent'; import superagent from '@cloudron/superagent';
const debug = debugModule('box:community'); const { log, trace } = logger('community');
const CLOUDRON_VERSIONS_FILE = 'CloudronVersions.json'; const CLOUDRON_VERSIONS_FILE = 'CloudronVersions.json';
@@ -125,7 +125,7 @@ async function downloadManifest(versionsUrl) {
if (!url.startsWith('https://')) throw new BoxError(BoxError.BAD_FIELD, 'versionsUrl must use https'); if (!url.startsWith('https://')) throw new BoxError(BoxError.BAD_FIELD, 'versionsUrl must use https');
if (!version) throw new BoxError(BoxError.BAD_FIELD, 'version is required in versionsUrl (format: url@version)'); if (!version) throw new BoxError(BoxError.BAD_FIELD, 'version is required in versionsUrl (format: url@version)');
debug(`downloading manifest from ${url} version ${version}`); log(`downloading manifest from ${url} version ${version}`);
const versionsRoot = await fetchVersionsRoot(url); const versionsRoot = await fetchVersionsRoot(url);
@@ -170,7 +170,7 @@ async function getAppUpdate(app, options) {
} }
async function downloadIcon(manifest) { async function downloadIcon(manifest) {
return await promiseRetry({ times: 10, interval: 5000, debug }, async function () { return await promiseRetry({ times: 10, interval: 5000, debug: log }, async function () {
const [networkError, response] = await safe(superagent.get(manifest.iconUrl) const [networkError, response] = await safe(superagent.get(manifest.iconUrl)
.timeout(60 * 1000) .timeout(60 * 1000)
.ok(() => true)); .ok(() => true));

54
src/cron.js
View File

@@ -6,7 +6,7 @@ import backupSites from './backupsites.js';
import cloudron from './cloudron.js'; import cloudron from './cloudron.js';
import constants from './constants.js'; import constants from './constants.js';
import { CronJob } from 'cron'; import { CronJob } from 'cron';
import debugModule from 'debug'; import logger from './logger.js';
import domains from './domains.js'; import domains from './domains.js';
import dyndns from './dyndns.js'; import dyndns from './dyndns.js';
import externalLdap from './externalldap.js'; import externalLdap from './externalldap.js';
@@ -24,7 +24,7 @@ import system from './system.js';
import updater from './updater.js'; import updater from './updater.js';
import util from 'node:util'; import util from 'node:util';
const debug = debugModule('box:cron'); const { log, trace } = logger('cron');
// IMPORTANT: These patterns are together because they spin tasks which acquire a lock // IMPORTANT: These patterns are together because they spin tasks which acquire a lock
// If the patterns overlap all the time, then the task may not ever get a chance to run! // If the patterns overlap all the time, then the task may not ever get a chance to run!
@@ -78,7 +78,7 @@ function getCronSeed() {
hour = Math.floor(24 * Math.random()); hour = Math.floor(24 * Math.random());
minute = Math.floor(60 * Math.random()); minute = Math.floor(60 * Math.random());
debug(`getCronSeed: writing new cron seed file with ${hour}:${minute} to ${paths.CRON_SEED_FILE}`); log(`getCronSeed: writing new cron seed file with ${hour}:${minute} to ${paths.CRON_SEED_FILE}`);
safe.fs.writeFileSync(paths.CRON_SEED_FILE, `${hour}:${minute}`); safe.fs.writeFileSync(paths.CRON_SEED_FILE, `${hour}:${minute}`);
} }
@@ -91,7 +91,7 @@ async function handleBackupScheduleChanged(site) {
const tz = await cloudron.getTimeZone(); const tz = await cloudron.getTimeZone();
debug(`handleBackupScheduleChanged: schedule ${site.schedule} (${tz})`); log(`handleBackupScheduleChanged: schedule ${site.schedule} (${tz})`);
if (gJobs.backups.has(site.id)) gJobs.backups.get(site.id).stop(); if (gJobs.backups.has(site.id)) gJobs.backups.get(site.id).stop();
gJobs.backups.delete(site.id); gJobs.backups.delete(site.id);
@@ -103,7 +103,7 @@ async function handleBackupScheduleChanged(site) {
onTick: async () => { onTick: async () => {
const t = await backupSites.get(site.id); const t = await backupSites.get(site.id);
if (!t) return; if (!t) return;
await safe(backupSites.startBackupTask(t, AuditSource.CRON), { debug }); await safe(backupSites.startBackupTask(t, AuditSource.CRON), { debug: log });
}, },
start: true, start: true,
timeZone: tz timeZone: tz
@@ -116,7 +116,7 @@ async function handleAutoupdatePatternChanged(pattern) {
const tz = await cloudron.getTimeZone(); const tz = await cloudron.getTimeZone();
debug(`autoupdatePatternChanged: pattern - ${pattern} (${tz})`); log(`autoupdatePatternChanged: pattern - ${pattern} (${tz})`);
if (gJobs.autoUpdater) gJobs.autoUpdater.stop(); if (gJobs.autoUpdater) gJobs.autoUpdater.stop();
gJobs.autoUpdater = null; gJobs.autoUpdater = null;
@@ -125,7 +125,7 @@ async function handleAutoupdatePatternChanged(pattern) {
gJobs.autoUpdater = CronJob.from({ gJobs.autoUpdater = CronJob.from({
cronTime: pattern, cronTime: pattern,
onTick: async () => await safe(updater.autoUpdate(AuditSource.CRON), { debug }), onTick: async () => await safe(updater.autoUpdate(AuditSource.CRON), { debug: log }),
start: true, start: true,
timeZone: tz timeZone: tz
}); });
@@ -134,7 +134,7 @@ async function handleAutoupdatePatternChanged(pattern) {
function handleDynamicDnsChanged(enabled) { function handleDynamicDnsChanged(enabled) {
assert.strictEqual(typeof enabled, 'boolean'); assert.strictEqual(typeof enabled, 'boolean');
debug('Dynamic DNS setting changed to %s', enabled); log('Dynamic DNS setting changed to %s', enabled);
if (gJobs.dynamicDns) gJobs.dynamicDns.stop(); if (gJobs.dynamicDns) gJobs.dynamicDns.stop();
gJobs.dynamicDns = null; gJobs.dynamicDns = null;
@@ -144,7 +144,7 @@ function handleDynamicDnsChanged(enabled) {
gJobs.dynamicDns = CronJob.from({ gJobs.dynamicDns = CronJob.from({
// until we can be smarter about actual IP changes, lets ensure it every 10minutes // until we can be smarter about actual IP changes, lets ensure it every 10minutes
cronTime: '00 */10 * * * *', cronTime: '00 */10 * * * *',
onTick: async () => { await safe(dyndns.refreshDns(AuditSource.CRON), { debug }); }, onTick: async () => { await safe(dyndns.refreshDns(AuditSource.CRON), { debug: log }); },
start: true start: true
}); });
} }
@@ -159,7 +159,7 @@ async function handleExternalLdapChanged(config) {
gJobs.externalLdapSyncer = CronJob.from({ gJobs.externalLdapSyncer = CronJob.from({
cronTime: '00 00 */4 * * *', // every 4 hours cronTime: '00 00 */4 * * *', // every 4 hours
onTick: async () => await safe(externalLdap.startSyncer(AuditSource.CRON), { debug }), onTick: async () => await safe(externalLdap.startSyncer(AuditSource.CRON), { debug: log }),
start: true start: true
}); });
} }
@@ -167,42 +167,42 @@ async function handleExternalLdapChanged(config) {
async function startJobs() { async function startJobs() {
const { hour, minute } = getCronSeed(); const { hour, minute } = getCronSeed();
debug(`startJobs: starting cron jobs with hour ${hour} and minute ${minute}`); log(`startJobs: starting cron jobs with hour ${hour} and minute ${minute}`);
gJobs.systemChecks = CronJob.from({ gJobs.systemChecks = CronJob.from({
cronTime: `00 ${minute} 2 * * *`, // once a day. if you change this interval, change the notification messages with correct duration cronTime: `00 ${minute} 2 * * *`, // once a day. if you change this interval, change the notification messages with correct duration
onTick: async () => await safe(system.runSystemChecks(), { debug }), onTick: async () => await safe(system.runSystemChecks(), { debug: log }),
start: true start: true
}); });
gJobs.mailStatusCheck = CronJob.from({ gJobs.mailStatusCheck = CronJob.from({
cronTime: `00 ${minute} 2 * * *`, // once a day. if you change this interval, change the notification messages with correct duration cronTime: `00 ${minute} 2 * * *`, // once a day. if you change this interval, change the notification messages with correct duration
onTick: async () => await safe(mail.checkStatus(), { debug }), onTick: async () => await safe(mail.checkStatus(), { debug: log }),
start: true start: true
}); });
gJobs.diskSpaceChecker = CronJob.from({ gJobs.diskSpaceChecker = CronJob.from({
cronTime: '00 30 * * * *', // every 30 minutes. if you change this interval, change the notification messages with correct duration cronTime: '00 30 * * * *', // every 30 minutes. if you change this interval, change the notification messages with correct duration
onTick: async () => await safe(system.checkDiskSpace(), { debug }), onTick: async () => await safe(system.checkDiskSpace(), { debug: log }),
start: true start: true
}); });
// this is run separately from the update itself so that the user can disable automatic updates but can still get a notification // this is run separately from the update itself so that the user can disable automatic updates but can still get a notification
gJobs.updateCheckerJob = CronJob.from({ gJobs.updateCheckerJob = CronJob.from({
cronTime: `00 ${minute} 1,5,9,13,17,21,23 * * *`, cronTime: `00 ${minute} 1,5,9,13,17,21,23 * * *`,
onTick: async () => await safe(updater.checkForUpdates({ stableOnly: true }), { debug }), onTick: async () => await safe(updater.checkForUpdates({ stableOnly: true }), { debug: log }),
start: true start: true
}); });
gJobs.cleanupTokens = CronJob.from({ gJobs.cleanupTokens = CronJob.from({
cronTime: '00 */30 * * * *', // every 30 minutes cronTime: '00 */30 * * * *', // every 30 minutes
onTick: async () => await safe(janitor.cleanupTokens(), { debug }), onTick: async () => await safe(janitor.cleanupTokens(), { debug: log }),
start: true start: true
}); });
gJobs.cleanupOidc = CronJob.from({ gJobs.cleanupOidc = CronJob.from({
cronTime: '00 10 * * * *', // every hour ten minutes past cronTime: '00 10 * * * *', // every hour ten minutes past
onTick: async () => await safe(oidcServer.cleanupExpired(), { debug }), onTick: async () => await safe(oidcServer.cleanupExpired(), { debug: log }),
start: true start: true
}); });
@@ -210,7 +210,7 @@ async function startJobs() {
cronTime: DEFAULT_CLEANUP_BACKUPS_PATTERN, cronTime: DEFAULT_CLEANUP_BACKUPS_PATTERN,
onTick: async () => { onTick: async () => {
for (const backupSite of await backupSites.list()) { for (const backupSite of await backupSites.list()) {
await safe(backupSites.startCleanupTask(backupSite, AuditSource.CRON), { debug }); await safe(backupSites.startCleanupTask(backupSite, AuditSource.CRON), { debug: log });
} }
}, },
start: true start: true
@@ -218,50 +218,50 @@ async function startJobs() {
gJobs.cleanupEventlog = CronJob.from({ gJobs.cleanupEventlog = CronJob.from({
cronTime: '00 */30 * * * *', // every 30 minutes cronTime: '00 */30 * * * *', // every 30 minutes
onTick: async () => await safe(eventlog.cleanup({ creationTime: new Date(Date.now() - 90 * 60 * 24 * 60 * 1000) }), { debug }), // 90 days ago onTick: async () => await safe(eventlog.cleanup({ creationTime: new Date(Date.now() - 90 * 60 * 24 * 60 * 1000) }), { debug: log }), // 90 days ago
start: true start: true
}); });
gJobs.dockerVolumeCleaner = CronJob.from({ gJobs.dockerVolumeCleaner = CronJob.from({
cronTime: '00 00 */12 * * *', // every 12 hours cronTime: '00 00 */12 * * *', // every 12 hours
onTick: async () => await safe(janitor.cleanupDockerVolumes(), { debug }), onTick: async () => await safe(janitor.cleanupDockerVolumes(), { debug: log }),
start: true start: true
}); });
gJobs.schedulerSync = CronJob.from({ gJobs.schedulerSync = CronJob.from({
cronTime: constants.TEST ? '*/10 * * * * *' : '00 */1 * * * *', // every minute cronTime: constants.TEST ? '*/10 * * * * *' : '00 */1 * * * *', // every minute
onTick: async () => await safe(scheduler.sync(), { debug }), onTick: async () => await safe(scheduler.sync(), { debug: log }),
start: true start: true
}); });
// randomized per Cloudron based on hourlySeed // randomized per Cloudron based on hourlySeed
gJobs.certificateRenew = CronJob.from({ gJobs.certificateRenew = CronJob.from({
cronTime: `00 10 ${hour} * * *`, cronTime: `00 10 ${hour} * * *`,
onTick: async () => await safe(reverseProxy.startRenewCerts({}, AuditSource.CRON), { debug }), onTick: async () => await safe(reverseProxy.startRenewCerts({}, AuditSource.CRON), { debug: log }),
start: true start: true
}); });
gJobs.checkDomainConfigs = CronJob.from({ gJobs.checkDomainConfigs = CronJob.from({
cronTime: `00 ${minute} 5 * * *`, // once a day cronTime: `00 ${minute} 5 * * *`, // once a day
onTick: async () => await safe(domains.checkConfigs(AuditSource.CRON), { debug }), onTick: async () => await safe(domains.checkConfigs(AuditSource.CRON), { debug: log }),
start: true start: true
}); });
gJobs.appHealthMonitor = CronJob.from({ gJobs.appHealthMonitor = CronJob.from({
cronTime: '*/10 * * * * *', // every 10 seconds cronTime: '*/10 * * * * *', // every 10 seconds
onTick: async () => await safe(appHealthMonitor.run(10), { debug }), // 10 is the max run time onTick: async () => await safe(appHealthMonitor.run(10), { debug: log }), // 10 is the max run time
start: true start: true
}); });
gJobs.collectStats = CronJob.from({ gJobs.collectStats = CronJob.from({
cronTime: '*/20 * * * * *', // every 20 seconds. if you change this, change carbon config cronTime: '*/20 * * * * *', // every 20 seconds. if you change this, change carbon config
onTick: async () => await safe(metrics.sendToGraphite(), { debug }), onTick: async () => await safe(metrics.sendToGraphite(), { debug: log }),
start: true start: true
}); });
gJobs.subscriptionChecker = CronJob.from({ gJobs.subscriptionChecker = CronJob.from({
cronTime: `00 ${minute} ${hour} * * *`, // once a day based on seed to randomize cronTime: `00 ${minute} ${hour} * * *`, // once a day based on seed to randomize
onTick: async () => await safe(appstore.checkSubscription(), { debug }), onTick: async () => await safe(appstore.checkSubscription(), { debug: log }),
start: true start: true
}); });
@@ -289,7 +289,7 @@ async function stopJobs() {
async function handleTimeZoneChanged(tz) { async function handleTimeZoneChanged(tz) {
assert.strictEqual(typeof tz, 'string'); assert.strictEqual(typeof tz, 'string');
debug('handleTimeZoneChanged: recreating all jobs'); log('handleTimeZoneChanged: recreating all jobs');
await stopJobs(); await stopJobs();
await scheduler.deleteJobs(); // have to re-create with new tz await scheduler.deleteJobs(); // have to re-create with new tz
await startJobs(); await startJobs();

18
src/dashboard.js
View File

@@ -4,7 +4,7 @@ import assert from 'node:assert';
import BoxError from './boxerror.js'; import BoxError from './boxerror.js';
import branding from './branding.js'; import branding from './branding.js';
import constants from './constants.js'; import constants from './constants.js';
import debugModule from 'debug'; import logger from './logger.js';
import dns from './dns.js'; import dns from './dns.js';
import externalLdap from './externalldap.js'; import externalLdap from './externalldap.js';
import eventlog from './eventlog.js'; import eventlog from './eventlog.js';
@@ -18,7 +18,7 @@ import system from './system.js';
import tasks from './tasks.js'; import tasks from './tasks.js';
import userDirectory from './user-directory.js'; import userDirectory from './user-directory.js';
const debug = debugModule('box:dashboard'); const { log, trace } = logger('dashboard');
async function getLocation() { async function getLocation() {
const domain = await settings.get(settings.DASHBOARD_DOMAIN_KEY); const domain = await settings.get(settings.DASHBOARD_DOMAIN_KEY);
@@ -33,7 +33,7 @@ async function setLocation(subdomain, domain) {
await settings.set(settings.DASHBOARD_SUBDOMAIN_KEY, subdomain); await settings.set(settings.DASHBOARD_SUBDOMAIN_KEY, subdomain);
await settings.set(settings.DASHBOARD_DOMAIN_KEY, domain); await settings.set(settings.DASHBOARD_DOMAIN_KEY, domain);
debug(`setLocation: ${domain || '<cleared>'}`); log(`setLocation: ${domain || '<cleared>'}`);
} }
async function clearLocation() { async function clearLocation() {
@@ -91,7 +91,7 @@ async function startPrepareLocation(domain, auditSource) {
assert.strictEqual(typeof domain, 'string'); assert.strictEqual(typeof domain, 'string');
assert.strictEqual(typeof auditSource, 'object'); assert.strictEqual(typeof auditSource, 'object');
debug(`prepareLocation: ${domain}`); log(`prepareLocation: ${domain}`);
if (constants.DEMO) throw new BoxError(BoxError.BAD_STATE, 'Not allowed in demo mode'); if (constants.DEMO) throw new BoxError(BoxError.BAD_STATE, 'Not allowed in demo mode');
@@ -105,7 +105,7 @@ async function startPrepareLocation(domain, auditSource) {
} }
const taskId = await tasks.add(tasks.TASK_PREPARE_DASHBOARD_LOCATION, [ constants.DASHBOARD_SUBDOMAIN, domain, auditSource ]); const taskId = await tasks.add(tasks.TASK_PREPARE_DASHBOARD_LOCATION, [ constants.DASHBOARD_SUBDOMAIN, domain, auditSource ]);
safe(tasks.startTask(taskId, {}), { debug }); // background safe(tasks.startTask(taskId, {}), { debug: log }); // background
return taskId; return taskId;
} }
@@ -115,7 +115,7 @@ async function setupLocation(subdomain, domain, auditSource) {
assert.strictEqual(typeof domain, 'string'); assert.strictEqual(typeof domain, 'string');
assert.strictEqual(typeof auditSource, 'object'); assert.strictEqual(typeof auditSource, 'object');
debug(`setupLocation: ${domain}`); log(`setupLocation: ${domain}`);
if (constants.DEMO) throw new BoxError(BoxError.BAD_STATE, 'Not allowed in demo mode'); if (constants.DEMO) throw new BoxError(BoxError.BAD_STATE, 'Not allowed in demo mode');
@@ -131,12 +131,12 @@ async function changeLocation(subdomain, domain, auditSource) {
const oldLocation = await getLocation(); const oldLocation = await getLocation();
await setupLocation(subdomain, domain, auditSource); await setupLocation(subdomain, domain, auditSource);
debug(`setupLocation: notifying appstore and platform of domain change to ${domain}`); log(`setupLocation: notifying appstore and platform of domain change to ${domain}`);
await eventlog.add(eventlog.ACTION_DASHBOARD_DOMAIN_UPDATE, auditSource, { subdomain, domain }); await eventlog.add(eventlog.ACTION_DASHBOARD_DOMAIN_UPDATE, auditSource, { subdomain, domain });
await safe(appstore.updateCloudron({ domain }), { debug }); await safe(appstore.updateCloudron({ domain }), { debug: log });
await platform.onDashboardLocationChanged(auditSource); await platform.onDashboardLocationChanged(auditSource);
await safe(reverseProxy.removeDashboardConfig(oldLocation.subdomain, oldLocation.domain), { debug }); await safe(reverseProxy.removeDashboardConfig(oldLocation.subdomain, oldLocation.domain), { debug: log });
} }
const _setLocation = setLocation; const _setLocation = setLocation;

14
src/database.js
View File

@@ -1,13 +1,13 @@
import assert from 'node:assert'; import assert from 'node:assert';
import BoxError from './boxerror.js'; import BoxError from './boxerror.js';
import constants from './constants.js'; import constants from './constants.js';
import debugModule from 'debug'; import logger from './logger.js';
import { execSync } from 'node:child_process'; import { execSync } from 'node:child_process';
import mysql from 'mysql2/promise'; import mysql from 'mysql2/promise';
import safe from 'safetydance'; import safe from 'safetydance';
import shellModule from './shell.js'; import shellModule from './shell.js';
const debug = debugModule('box:database'); const { log, trace } = logger('database');
const shell = shellModule('database'); const shell = shellModule('database');
let gConnectionPool = null; let gConnectionPool = null;
@@ -23,9 +23,9 @@ const gDatabase = {
async function uninitialize() { async function uninitialize() {
if (!gConnectionPool) return; if (!gConnectionPool) return;
await safe(gConnectionPool.end(), { debug }); await safe(gConnectionPool.end(), { debug: log });
gConnectionPool = null; gConnectionPool = null;
debug('pool closed'); log('pool closed');
} }
async function query(...args) { async function query(...args) {
@@ -53,7 +53,7 @@ async function transaction(queries) {
connection.release(); // no await! connection.release(); // no await!
return results; return results;
} catch (txError) { } catch (txError) {
await safe(connection.rollback(), { debug }); await safe(connection.rollback(), { debug: log });
connection.release(); // no await! connection.release(); // no await!
throw new BoxError(BoxError.DATABASE_ERROR, txError, { sqlCode: txError.code, sqlMessage: txError.sqlMessage || null }); throw new BoxError(BoxError.DATABASE_ERROR, txError, { sqlCode: txError.code, sqlMessage: txError.sqlMessage || null });
} }
@@ -95,7 +95,7 @@ async function initialize() {
// a crypto.randomUUID is 36 in length. so the value below provides for roughly 10k users // a crypto.randomUUID is 36 in length. so the value below provides for roughly 10k users
await conn.query('SET SESSION group_concat_max_len = 360000'); await conn.query('SET SESSION group_concat_max_len = 360000');
} catch (error) { } catch (error) {
debug(`failed to init new db connection ${connection.threadId}:`, error); // only log. we will let the app handle the exception when it calls query()/transaction() log(`failed to init new db connection ${connection.threadId}:`, error); // only log. we will let the app handle the exception when it calls query()/transaction()
} }
}); });
} }
@@ -129,7 +129,7 @@ async function runInTransaction(callback) {
connection.release(); // no await! connection.release(); // no await!
return result; return result;
} catch (txError) { } catch (txError) {
await safe(connection.rollback(), { debug }); await safe(connection.rollback(), { debug: log });
connection.release(); // no await! connection.release(); // no await!
throw new BoxError(BoxError.DATABASE_ERROR, txError, { sqlCode: txError.code, sqlMessage: txError.sqlMessage || null }); throw new BoxError(BoxError.DATABASE_ERROR, txError, { sqlCode: txError.code, sqlMessage: txError.sqlMessage || null });
} }

8
src/df.js
View File

@@ -1,10 +1,10 @@
import assert from 'node:assert'; import assert from 'node:assert';
import BoxError from './boxerror.js'; import BoxError from './boxerror.js';
import debugModule from 'debug'; import logger from './logger.js';
import safe from 'safetydance'; import safe from 'safetydance';
import shellModule from './shell.js'; import shellModule from './shell.js';
const debug = debugModule('box:df'); const { log, trace } = logger('df');
const shell = shellModule('df'); const shell = shellModule('df');
@@ -35,7 +35,7 @@ function parseLine(line) {
async function filesystems() { async function filesystems() {
const [error, output] = await safe(shell.spawn('df', ['-B1', '--output=source,fstype,size,used,avail,pcent,target'], { encoding: 'utf8', timeout: 5000 })); const [error, output] = await safe(shell.spawn('df', ['-B1', '--output=source,fstype,size,used,avail,pcent,target'], { encoding: 'utf8', timeout: 5000 }));
if (error) { if (error) {
debug(`filesystems: df command failed. error: ${error}\n stdout: ${error.stdout}\n stderr: ${error.stderr}`); log(`filesystems: df command failed. error: ${error}\n stdout: ${error.stdout}\n stderr: ${error.stderr}`);
throw new BoxError(BoxError.FS_ERROR, `Error running df: ${error.message}`); throw new BoxError(BoxError.FS_ERROR, `Error running df: ${error.message}`);
} }
@@ -54,7 +54,7 @@ async function file(filename) {
const [error, output] = await safe(shell.spawn('df', ['-B1', '--output=source,fstype,size,used,avail,pcent,target', filename], { encoding: 'utf8', timeout: 5000 })); const [error, output] = await safe(shell.spawn('df', ['-B1', '--output=source,fstype,size,used,avail,pcent,target', filename], { encoding: 'utf8', timeout: 5000 }));
if (error) { if (error) {
debug(`file: df command failed. error: ${error}\n stdout: ${error.stdout}\n stderr: ${error.stderr}`); log(`file: df command failed. error: ${error}\n stdout: ${error.stdout}\n stderr: ${error.stderr}`);
throw new BoxError(BoxError.FS_ERROR, `Error running df: ${error.message}`); throw new BoxError(BoxError.FS_ERROR, `Error running df: ${error.message}`);
} }

34
src/directoryserver.js
View File

@@ -2,7 +2,7 @@ import assert from 'node:assert';
import AuditSource from './auditsource.js'; import AuditSource from './auditsource.js';
import BoxError from './boxerror.js'; import BoxError from './boxerror.js';
import constants from './constants.js'; import constants from './constants.js';
import debugModule from 'debug'; import logger from './logger.js';
import eventlog from './eventlog.js'; import eventlog from './eventlog.js';
import ipaddr from './ipaddr.js'; import ipaddr from './ipaddr.js';
import groups from './groups.js'; import groups from './groups.js';
@@ -16,7 +16,7 @@ import shellModule from './shell.js';
import users from './users.js'; import users from './users.js';
import util from 'node:util'; import util from 'node:util';
const debug = debugModule('box:directoryserver'); const { log, trace } = logger('directoryserver');
const shell = shellModule('directoryserver'); const shell = shellModule('directoryserver');
@@ -57,7 +57,7 @@ async function validateConfig(config) {
} }
async function authorize(req, res, next) { async function authorize(req, res, next) {
debug('authorize: ', req.connection.ldap.bindDN.toString()); log('authorize: ', req.connection.ldap.bindDN.toString());
// this is for connection attempts without previous bind // this is for connection attempts without previous bind
if (req.connection.ldap.bindDN.equals('cn=anonymous')) return next(new ldap.InsufficientAccessRightsError()); if (req.connection.ldap.bindDN.equals('cn=anonymous')) return next(new ldap.InsufficientAccessRightsError());
@@ -69,7 +69,7 @@ async function authorize(req, res, next) {
} }
async function maybeRootDSE(req, res, next) { async function maybeRootDSE(req, res, next) {
debug(`maybeRootDSE: requested with scope:${req.scope} dn:${req.dn.toString()}`); log(`maybeRootDSE: requested with scope:${req.scope} dn:${req.dn.toString()}`);
if (req.scope !== 'base') return next(new ldap.NoSuchObjectError()); // per the spec, rootDSE search require base scope if (req.scope !== 'base') return next(new ldap.NoSuchObjectError()); // per the spec, rootDSE search require base scope
if (!req.dn || req.dn.toString() !== '') return next(new ldap.NoSuchObjectError()); if (!req.dn || req.dn.toString() !== '') return next(new ldap.NoSuchObjectError());
@@ -126,7 +126,7 @@ async function userAuth(req, res, next) {
async function stop() { async function stop() {
if (!gServer) return; if (!gServer) return;
debug('stopping server'); log('stopping server');
await util.promisify(gServer.close.bind(gServer))(); await util.promisify(gServer.close.bind(gServer))();
gServer = null; gServer = null;
@@ -192,7 +192,7 @@ function finalSend(results, req, res, next) {
// Will attach req.user if successful // Will attach req.user if successful
async function userSearch(req, res, next) { async function userSearch(req, res, next) {
debug('user search: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id); log('user search: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id);
const [error, allUsers] = await safe(users.list()); const [error, allUsers] = await safe(users.list());
if (error) return next(new ldap.OperationsError(error.message)); if (error) return next(new ldap.OperationsError(error.message));
@@ -248,7 +248,7 @@ async function userSearch(req, res, next) {
} }
async function groupSearch(req, res, next) { async function groupSearch(req, res, next) {
debug('group search: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id); log('group search: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id);
const [error, allUsers] = await safe(users.list()); const [error, allUsers] = await safe(users.list());
if (error) return next(new ldap.OperationsError(error.message)); if (error) return next(new ldap.OperationsError(error.message));
@@ -292,10 +292,10 @@ async function start() {
const logger = { const logger = {
trace: NOOP, trace: NOOP,
debug: NOOP, debug: NOOP,
info: debug, info: log,
warn: debug, warn: log,
error: debug, error: log,
fatal: debug fatal: log
}; };
gCertificate = await reverseProxy.getDirectoryServerCertificate(); gCertificate = await reverseProxy.getDirectoryServerCertificate();
@@ -307,11 +307,11 @@ async function start() {
}); });
gServer.on('error', function (error) { gServer.on('error', function (error) {
debug('server startup error: %o', error); log('server startup error: %o', error);
}); });
gServer.bind('ou=system,dc=cloudron', async function(req, res, next) { gServer.bind('ou=system,dc=cloudron', async function(req, res, next) {
debug('system bind: %s (from %s)', req.dn.toString(), req.connection.ldap.id); log('system bind: %s (from %s)', req.dn.toString(), req.connection.ldap.id);
const config = await getConfig(); const config = await getConfig();
@@ -340,11 +340,11 @@ async function start() {
// just log that an attempt was made to unknown route, this helps a lot during app packaging // just log that an attempt was made to unknown route, this helps a lot during app packaging
gServer.use(function(req, res, next) { gServer.use(function(req, res, next) {
debug('not handled: dn %s, scope %s, filter %s (from %s)', req.dn ? req.dn.toString() : '-', req.scope, req.filter ? req.filter.toString() : '-', req.connection.ldap.id); log('not handled: dn %s, scope %s, filter %s (from %s)', req.dn ? req.dn.toString() : '-', req.scope, req.filter ? req.filter.toString() : '-', req.connection.ldap.id);
return next(); return next();
}); });
debug(`starting server on port ${constants.USER_DIRECTORY_LDAPS_PORT}`); log(`starting server on port ${constants.USER_DIRECTORY_LDAPS_PORT}`);
await util.promisify(gServer.listen.bind(gServer))(constants.USER_DIRECTORY_LDAPS_PORT, '::'); await util.promisify(gServer.listen.bind(gServer))(constants.USER_DIRECTORY_LDAPS_PORT, '::');
} }
@@ -397,11 +397,11 @@ async function checkCertificate() {
const certificate = await reverseProxy.getDirectoryServerCertificate(); const certificate = await reverseProxy.getDirectoryServerCertificate();
if (certificate.cert === gCertificate.cert) { if (certificate.cert === gCertificate.cert) {
debug('checkCertificate: certificate has not changed'); log('checkCertificate: certificate has not changed');
return; return;
} }
debug('checkCertificate: certificate changed. restarting'); log('checkCertificate: certificate changed. restarting');
await stop(); await stop();
await start(); await start();
} }

28
src/dns.js
View File

@@ -3,7 +3,7 @@ import assert from 'node:assert';
import BoxError from './boxerror.js'; import BoxError from './boxerror.js';
import constants from './constants.js'; import constants from './constants.js';
import dashboard from './dashboard.js'; import dashboard from './dashboard.js';
import debugModule from 'debug'; import logger from './logger.js';
import domains from './domains.js'; import domains from './domains.js';
import ipaddr from './ipaddr.js'; import ipaddr from './ipaddr.js';
import mail from './mail.js'; import mail from './mail.js';
@@ -36,7 +36,7 @@ import dnsOvh from './dns/ovh.js';
import dnsPorkbun from './dns/porkbun.js'; import dnsPorkbun from './dns/porkbun.js';
import dnsWildcard from './dns/wildcard.js'; import dnsWildcard from './dns/wildcard.js';
const debug = debugModule('box:dns'); const { log, trace } = logger('dns');
const DNS_PROVIDERS = { const DNS_PROVIDERS = {
@@ -140,7 +140,7 @@ async function upsertDnsRecords(subdomain, domain, type, values) {
assert.strictEqual(typeof type, 'string'); assert.strictEqual(typeof type, 'string');
assert(Array.isArray(values)); assert(Array.isArray(values));
debug(`upsertDnsRecords: subdomain:${subdomain} domain:${domain} type:${type} values:${JSON.stringify(values)}`); log(`upsertDnsRecords: subdomain:${subdomain} domain:${domain} type:${type} values:${JSON.stringify(values)}`);
const domainObject = await domains.get(domain); const domainObject = await domains.get(domain);
await api(domainObject.provider).upsert(domainObject, subdomain, type, values); await api(domainObject.provider).upsert(domainObject, subdomain, type, values);
@@ -152,7 +152,7 @@ async function removeDnsRecords(subdomain, domain, type, values) {
assert.strictEqual(typeof type, 'string'); assert.strictEqual(typeof type, 'string');
assert(Array.isArray(values)); assert(Array.isArray(values));
debug(`removeDnsRecords: subdomain:${subdomain} domain:${domain} type:${type} values:${JSON.stringify(values)}`); log(`removeDnsRecords: subdomain:${subdomain} domain:${domain} type:${type} values:${JSON.stringify(values)}`);
const domainObject = await domains.get(domain); const domainObject = await domains.get(domain);
const [error] = await safe(api(domainObject.provider).del(domainObject, subdomain, type, values)); const [error] = await safe(api(domainObject.provider).del(domainObject, subdomain, type, values));
@@ -220,7 +220,7 @@ async function registerLocation(location, options, recordType, recordValue) {
const [getError, values] = await safe(getDnsRecords(location.subdomain, location.domain, recordType)); const [getError, values] = await safe(getDnsRecords(location.subdomain, location.domain, recordType));
if (getError) { if (getError) {
const retryable = getError.reason !== BoxError.ACCESS_DENIED && getError.reason !== BoxError.NOT_FOUND; // NOT_FOUND is when zone is not found const retryable = getError.reason !== BoxError.ACCESS_DENIED && getError.reason !== BoxError.NOT_FOUND; // NOT_FOUND is when zone is not found
debug(`registerLocation: Get error. retryable: ${retryable}. ${getError.message}`); log(`registerLocation: Get error. retryable: ${retryable}. ${getError.message}`);
throw new BoxError(getError.reason, `${location.domain}: ${getError.message}`, { domain: location, retryable }); throw new BoxError(getError.reason, `${location.domain}: ${getError.message}`, { domain: location, retryable });
} }
@@ -232,7 +232,7 @@ async function registerLocation(location, options, recordType, recordValue) {
const [upsertError] = await safe(upsertDnsRecords(location.subdomain, location.domain, recordType, [ recordValue ])); const [upsertError] = await safe(upsertDnsRecords(location.subdomain, location.domain, recordType, [ recordValue ]));
if (upsertError) { if (upsertError) {
const retryable = upsertError.reason === BoxError.BUSY || upsertError.reason === BoxError.EXTERNAL_ERROR; const retryable = upsertError.reason === BoxError.BUSY || upsertError.reason === BoxError.EXTERNAL_ERROR;
debug(`registerLocation: Upsert error. retryable: ${retryable}. ${upsertError.message}`); log(`registerLocation: Upsert error. retryable: ${retryable}. ${upsertError.message}`);
throw new BoxError(BoxError.EXTERNAL_ERROR, `${location.domain}: ${getError.message}`, { domain: location, retryable }); throw new BoxError(BoxError.EXTERNAL_ERROR, `${location.domain}: ${getError.message}`, { domain: location, retryable });
} }
} }
@@ -242,7 +242,7 @@ async function registerLocations(locations, options, progressCallback) {
assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof options, 'object');
assert.strictEqual(typeof progressCallback, 'function'); assert.strictEqual(typeof progressCallback, 'function');
debug(`registerLocations: Will register ${JSON.stringify(locations)} with options ${JSON.stringify(options)}`); log(`registerLocations: Will register ${JSON.stringify(locations)} with options ${JSON.stringify(options)}`);
const ipv4 = await network.getIPv4(); const ipv4 = await network.getIPv4();
const ipv6 = await network.getIPv6(); const ipv6 = await network.getIPv6();
@@ -250,12 +250,12 @@ async function registerLocations(locations, options, progressCallback) {
for (const location of locations) { for (const location of locations) {
progressCallback({ message: `Registering location ${fqdn(location.subdomain, location.domain)}` }); progressCallback({ message: `Registering location ${fqdn(location.subdomain, location.domain)}` });
await promiseRetry({ times: 200, interval: 5000, debug, retry: (error) => error.retryable }, async function () { await promiseRetry({ times: 200, interval: 5000, debug: log, retry: (error) => error.retryable }, async function () {
// cname records cannot co-exist with other records // cname records cannot co-exist with other records
const [getError, values] = await safe(getDnsRecords(location.subdomain, location.domain, 'CNAME')); const [getError, values] = await safe(getDnsRecords(location.subdomain, location.domain, 'CNAME'));
if (!getError && values.length === 1) { if (!getError && values.length === 1) {
if (!options.overwriteDns) throw new BoxError(BoxError.ALREADY_EXISTS, 'DNS CNAME record already exists', { domain: location, retryable: false }); if (!options.overwriteDns) throw new BoxError(BoxError.ALREADY_EXISTS, 'DNS CNAME record already exists', { domain: location, retryable: false });
debug(`registerLocations: removing CNAME record of ${fqdn(location.subdomain, location.domain)}`); log(`registerLocations: removing CNAME record of ${fqdn(location.subdomain, location.domain)}`);
await removeDnsRecords(location.subdomain, location.domain, 'CNAME', values); await removeDnsRecords(location.subdomain, location.domain, 'CNAME', values);
} }
@@ -263,14 +263,14 @@ async function registerLocations(locations, options, progressCallback) {
await registerLocation(location, options, 'A', ipv4); await registerLocation(location, options, 'A', ipv4);
} else { } else {
const [aError, aValues] = await safe(getDnsRecords(location.subdomain, location.domain, 'A')); const [aError, aValues] = await safe(getDnsRecords(location.subdomain, location.domain, 'A'));
if (!aError && aValues.length) await safe(removeDnsRecords(location.subdomain, location.domain, 'A', aValues), { debug }); if (!aError && aValues.length) await safe(removeDnsRecords(location.subdomain, location.domain, 'A', aValues), { debug: log });
} }
if (ipv6) { if (ipv6) {
await registerLocation(location, options, 'AAAA', ipv6); await registerLocation(location, options, 'AAAA', ipv6);
} else { } else {
const [aaaaError, aaaaValues] = await safe(getDnsRecords(location.subdomain, location.domain, 'AAAA')); const [aaaaError, aaaaValues] = await safe(getDnsRecords(location.subdomain, location.domain, 'AAAA'));
if (!aaaaError && aaaaValues.length) await safe(removeDnsRecords(location.subdomain, location.domain, 'AAAA', aaaaValues), { debug }); if (!aaaaError && aaaaValues.length) await safe(removeDnsRecords(location.subdomain, location.domain, 'AAAA', aaaaValues), { debug: log });
} }
}); });
} }
@@ -285,7 +285,7 @@ async function unregisterLocation(location, recordType, recordValue) {
if (!error || error.reason === BoxError.NOT_FOUND) return; if (!error || error.reason === BoxError.NOT_FOUND) return;
const retryable = error.reason === BoxError.BUSY || error.reason === BoxError.EXTERNAL_ERROR; const retryable = error.reason === BoxError.BUSY || error.reason === BoxError.EXTERNAL_ERROR;
debug(`unregisterLocation: Error unregistering location ${recordType}. retryable: ${retryable}. ${error.message}`); log(`unregisterLocation: Error unregistering location ${recordType}. retryable: ${retryable}. ${error.message}`);
throw new BoxError(BoxError.EXTERNAL_ERROR, `${location.domain}: ${error.message}`, { domain: location, retryable }); throw new BoxError(BoxError.EXTERNAL_ERROR, `${location.domain}: ${error.message}`, { domain: location, retryable });
} }
@@ -300,7 +300,7 @@ async function unregisterLocations(locations, progressCallback) {
for (const location of locations) { for (const location of locations) {
progressCallback({ message: `Unregistering location: ${location.subdomain ? (location.subdomain + '.') : ''}${location.domain}` }); progressCallback({ message: `Unregistering location: ${location.subdomain ? (location.subdomain + '.') : ''}${location.domain}` });
await promiseRetry({ times: 30, interval: 5000, debug, retry: (error) => error.retryable }, async function () { await promiseRetry({ times: 30, interval: 5000, debug: log, retry: (error) => error.retryable }, async function () {
if (ipv4) await unregisterLocation(location, 'A', ipv4); if (ipv4) await unregisterLocation(location, 'A', ipv4);
if (ipv6) await unregisterLocation(location, 'AAAA', ipv6); if (ipv6) await unregisterLocation(location, 'AAAA', ipv6);
}); });
@@ -364,7 +364,7 @@ async function startSyncDnsRecords(options) {
assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof options, 'object');
const taskId = await tasks.add(tasks.TASK_SYNC_DNS_RECORDS, [ options ]); const taskId = await tasks.add(tasks.TASK_SYNC_DNS_RECORDS, [ options ]);
safe(tasks.startTask(taskId, {}), { debug }); // background safe(tasks.startTask(taskId, {}), { debug: log }); // background
return taskId; return taskId;
} }

20
src/dns/bunny.js
View File

@@ -1,14 +1,14 @@
import assert from 'node:assert'; import assert from 'node:assert';
import BoxError from '../boxerror.js'; import BoxError from '../boxerror.js';
import constants from '../constants.js'; import constants from '../constants.js';
import debugModule from 'debug'; import logger from '../logger.js';
import dig from '../dig.js'; import dig from '../dig.js';
import dns from '../dns.js'; import dns from '../dns.js';
import safe from 'safetydance'; import safe from 'safetydance';
import superagent from '@cloudron/superagent'; import superagent from '@cloudron/superagent';
import waitForDns from './waitfordns.js'; import waitForDns from './waitfordns.js';
const debug = debugModule('box:dns/bunny'); const { log, trace } = logger('dns/bunny');
const BUNNY_API = 'https://api.bunny.net'; const BUNNY_API = 'https://api.bunny.net';
@@ -61,7 +61,7 @@ async function getDnsRecords(domainConfig, zoneName, name, type) {
assert.strictEqual(typeof name, 'string'); assert.strictEqual(typeof name, 'string');
assert.strictEqual(typeof type, 'string'); assert.strictEqual(typeof type, 'string');
debug(`get: ${name} in zone ${zoneName} of type ${type}`); log(`get: ${name} in zone ${zoneName} of type ${type}`);
const zoneId = await getZoneId(domainConfig, zoneName); const zoneId = await getZoneId(domainConfig, zoneName);
const [error, response] = await safe(superagent.get(`${BUNNY_API}/dnszone/${zoneId}`) const [error, response] = await safe(superagent.get(`${BUNNY_API}/dnszone/${zoneId}`)
@@ -87,7 +87,7 @@ async function upsert(domainObject, location, type, values) {
zoneName = domainObject.zoneName, zoneName = domainObject.zoneName,
name = dns.getName(domainObject, location, type) || ''; name = dns.getName(domainObject, location, type) || '';
debug(`upsert: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`); log(`upsert: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
const zoneId = await getZoneId(domainConfig, zoneName); const zoneId = await getZoneId(domainConfig, zoneName);
const records = await getDnsRecords(domainConfig, zoneName, name, type); const records = await getDnsRecords(domainConfig, zoneName, name, type);
@@ -148,10 +148,10 @@ async function upsert(domainObject, location, type, values) {
.timeout(30 * 1000) .timeout(30 * 1000)
.ok(() => true)); .ok(() => true));
if (error) debug(`upsert: error removing record ${records[j].id}: ${error.message}`); if (error) log(`upsert: error removing record ${records[j].id}: ${error.message}`);
} }
debug('upsert: completed with recordIds:%j', recordIds); log('upsert: completed with recordIds:%j', recordIds);
} }
async function get(domainObject, location, type) { async function get(domainObject, location, type) {
@@ -177,7 +177,7 @@ async function del(domainObject, location, type, values) {
zoneName = domainObject.zoneName, zoneName = domainObject.zoneName,
name = dns.getName(domainObject, location, type) || ''; name = dns.getName(domainObject, location, type) || '';
debug(`del: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`); log(`del: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
const zoneId = await getZoneId(domainConfig, zoneName); const zoneId = await getZoneId(domainConfig, zoneName);
const records = await getDnsRecords(domainConfig, zoneName, name, type); const records = await getDnsRecords(domainConfig, zoneName, name, type);
@@ -232,17 +232,17 @@ async function verifyDomainConfig(domainObject) {
if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers'); if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers');
if (!nameservers.every(function (n) { return n.toLowerCase().indexOf('.bunny.net') !== -1; })) { if (!nameservers.every(function (n) { return n.toLowerCase().indexOf('.bunny.net') !== -1; })) {
debug('verifyDomainConfig: %j does not contain Bunny NS', nameservers); log('verifyDomainConfig: %j does not contain Bunny NS', nameservers);
if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Bunny'); if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Bunny');
} }
const location = 'cloudrontestdns'; const location = 'cloudrontestdns';
await upsert(domainObject, location, 'A', [ ip ]); await upsert(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record added'); log('verifyDomainConfig: Test A record added');
await del(domainObject, location, 'A', [ ip ]); await del(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record removed again'); log('verifyDomainConfig: Test A record removed again');
return credentials; return credentials;
} }

24
src/dns/cloudflare.js
View File

@@ -1,7 +1,7 @@
import assert from 'node:assert'; import assert from 'node:assert';
import BoxError from '../boxerror.js'; import BoxError from '../boxerror.js';
import constants from '../constants.js'; import constants from '../constants.js';
import debugModule from 'debug'; import logger from '../logger.js';
import dig from '../dig.js'; import dig from '../dig.js';
import dns from '../dns.js'; import dns from '../dns.js';
import safe from 'safetydance'; import safe from 'safetydance';
@@ -9,7 +9,7 @@ import superagent from '@cloudron/superagent';
import waitForDns from './waitfordns.js'; import waitForDns from './waitfordns.js';
import _ from '../underscore.js'; import _ from '../underscore.js';
const debug = debugModule('box:dns/cloudflare'); const { log, trace } = logger('dns/cloudflare');
// we are using latest v4 stable API https://api.cloudflare.com/#getting-started-endpoints // we are using latest v4 stable API https://api.cloudflare.com/#getting-started-endpoints
@@ -107,7 +107,7 @@ async function upsert(domainObject, location, type, values) {
zoneName = domainObject.zoneName, zoneName = domainObject.zoneName,
fqdn = dns.fqdn(location, domainObject.domain); fqdn = dns.fqdn(location, domainObject.domain);
debug('upsert: %s for zone %s of type %s with values %j', fqdn, zoneName, type, values); log('upsert: %s for zone %s of type %s with values %j', fqdn, zoneName, type, values);
const zone = await getZoneByName(domainConfig, zoneName); const zone = await getZoneByName(domainConfig, zoneName);
const zoneId = zone.id; const zoneId = zone.id;
@@ -138,7 +138,7 @@ async function upsert(domainObject, location, type, values) {
data.proxied = !!domainConfig.defaultProxyStatus; // note that cloudflare will error if proxied is set for wrong record type or IP. only set at install time data.proxied = !!domainConfig.defaultProxyStatus; // note that cloudflare will error if proxied is set for wrong record type or IP. only set at install time
} }
debug(`upsert: Adding new record fqdn: ${fqdn}, zoneName: ${zoneName} proxied: ${data.proxied}`); log(`upsert: Adding new record fqdn: ${fqdn}, zoneName: ${zoneName} proxied: ${data.proxied}`);
const [error, response] = await safe(createRequest('POST', `${CLOUDFLARE_ENDPOINT}/zones/${zoneId}/dns_records`, domainConfig) const [error, response] = await safe(createRequest('POST', `${CLOUDFLARE_ENDPOINT}/zones/${zoneId}/dns_records`, domainConfig)
.send(data)); .send(data));
@@ -147,7 +147,7 @@ async function upsert(domainObject, location, type, values) {
} else { // replace existing record } else { // replace existing record
data.proxied = records[i].proxied; // preserve proxied parameter data.proxied = records[i].proxied; // preserve proxied parameter
debug(`upsert: Updating existing record fqdn: ${fqdn}, zoneName: ${zoneName} proxied: ${data.proxied}`); log(`upsert: Updating existing record fqdn: ${fqdn}, zoneName: ${zoneName} proxied: ${data.proxied}`);
const [error, response] = await safe(createRequest('PUT', `${CLOUDFLARE_ENDPOINT}/zones/${zoneId}/dns_records/${records[i].id}`, domainConfig) const [error, response] = await safe(createRequest('PUT', `${CLOUDFLARE_ENDPOINT}/zones/${zoneId}/dns_records/${records[i].id}`, domainConfig)
.send(data)); .send(data));
@@ -160,7 +160,7 @@ async function upsert(domainObject, location, type, values) {
for (let j = values.length + 1; j < records.length; j++) { for (let j = values.length + 1; j < records.length; j++) {
const [error] = await safe(createRequest('DELETE', `${CLOUDFLARE_ENDPOINT}/zones/${zoneId}/dns_records/${records[j].id}`, domainConfig)); const [error] = await safe(createRequest('DELETE', `${CLOUDFLARE_ENDPOINT}/zones/${zoneId}/dns_records/${records[j].id}`, domainConfig));
if (error) debug(`upsert: error removing record ${records[j].id}: ${error.message}`); if (error) log(`upsert: error removing record ${records[j].id}: ${error.message}`);
} }
} }
@@ -195,7 +195,7 @@ async function del(domainObject, location, type, values) {
if (result.length === 0) return; if (result.length === 0) return;
const tmp = result.filter(function (record) { return values.some(function (value) { return value === record.content; }); }); const tmp = result.filter(function (record) { return values.some(function (value) { return value === record.content; }); });
debug('del: %j', tmp); log('del: %j', tmp);
if (tmp.length === 0) return; if (tmp.length === 0) return;
@@ -217,7 +217,7 @@ async function wait(domainObject, subdomain, type, value, options) {
zoneName = domainObject.zoneName, zoneName = domainObject.zoneName,
fqdn = dns.fqdn(subdomain, domainObject.domain); fqdn = dns.fqdn(subdomain, domainObject.domain);
debug('wait: %s for zone %s of type %s', fqdn, zoneName, type); log('wait: %s for zone %s of type %s', fqdn, zoneName, type);
const zone = await getZoneByName(domainConfig, zoneName); const zone = await getZoneByName(domainConfig, zoneName);
const zoneId = zone.id; const zoneId = zone.id;
@@ -227,7 +227,7 @@ async function wait(domainObject, subdomain, type, value, options) {
if (!dnsRecords[0].proxied) return await waitForDns(fqdn, domainObject.zoneName, type, value, options); if (!dnsRecords[0].proxied) return await waitForDns(fqdn, domainObject.zoneName, type, value, options);
debug('wait: skipping wait of proxied domain'); log('wait: skipping wait of proxied domain');
// maybe we can check for dns to be cloudflare IPs? https://api.cloudflare.com/#cloudflare-ips-cloudflare-ip-details // maybe we can check for dns to be cloudflare IPs? https://api.cloudflare.com/#cloudflare-ips-cloudflare-ip-details
} }
@@ -268,17 +268,17 @@ async function verifyDomainConfig(domainObject) {
const zone = await getZoneByName(domainConfig, zoneName); const zone = await getZoneByName(domainConfig, zoneName);
if (!_.isEqual(zone.name_servers.sort(), nameservers.sort())) { if (!_.isEqual(zone.name_servers.sort(), nameservers.sort())) {
debug('verifyDomainConfig: %j and %j do not match', nameservers, zone.name_servers); log('verifyDomainConfig: %j and %j do not match', nameservers, zone.name_servers);
if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Cloudflare'); if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Cloudflare');
} }
const location = 'cloudrontestdns'; const location = 'cloudrontestdns';
await upsert(domainObject, location, 'A', [ ip ]); await upsert(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record added'); log('verifyDomainConfig: Test A record added');
await del(domainObject, location, 'A', [ ip ]); await del(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record removed again'); log('verifyDomainConfig: Test A record removed again');
return sanitizedConfig; return sanitizedConfig;
} }

10
src/dns/desec.js
View File

@@ -1,7 +1,7 @@
import assert from 'node:assert'; import assert from 'node:assert';
import constants from '../constants.js'; import constants from '../constants.js';
import BoxError from '../boxerror.js'; import BoxError from '../boxerror.js';
import debugModule from 'debug'; import logger from '../logger.js';
import dig from '../dig.js'; import dig from '../dig.js';
import dns from '../dns.js'; import dns from '../dns.js';
import safe from 'safetydance'; import safe from 'safetydance';
@@ -9,7 +9,7 @@ import timers from 'timers/promises';
import superagent from '@cloudron/superagent'; import superagent from '@cloudron/superagent';
import waitForDns from './waitfordns.js'; import waitForDns from './waitfordns.js';
const debug = debugModule('box:dns/desec'); const { log, trace } = logger('dns/desec');
const DESEC_ENDPOINT = 'https://desec.io/api/v1'; const DESEC_ENDPOINT = 'https://desec.io/api/v1';
@@ -143,17 +143,17 @@ async function verifyDomainConfig(domainObject) {
if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers'); if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers');
if (!nameservers.every(function (n) { return n.toLowerCase().indexOf('.desec.') !== -1; })) { if (!nameservers.every(function (n) { return n.toLowerCase().indexOf('.desec.') !== -1; })) {
debug('verifyDomainConfig: %j does not contains deSEC NS', nameservers); log('verifyDomainConfig: %j does not contains deSEC NS', nameservers);
if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to deSEC'); if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to deSEC');
} }
const location = 'cloudrontestdns'; const location = 'cloudrontestdns';
await upsert(domainObject, location, 'A', [ ip ]); await upsert(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record added'); log('verifyDomainConfig: Test A record added');
await del(domainObject, location, 'A', [ ip ]); await del(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record removed again'); log('verifyDomainConfig: Test A record removed again');
return credentials; return credentials;
} }

18
src/dns/digitalocean.js
View File

@@ -1,14 +1,14 @@
import assert from 'node:assert'; import assert from 'node:assert';
import BoxError from '../boxerror.js'; import BoxError from '../boxerror.js';
import constants from '../constants.js'; import constants from '../constants.js';
import debugModule from 'debug'; import logger from '../logger.js';
import dig from '../dig.js'; import dig from '../dig.js';
import dns from '../dns.js'; import dns from '../dns.js';
import safe from 'safetydance'; import safe from 'safetydance';
import superagent from '@cloudron/superagent'; import superagent from '@cloudron/superagent';
import waitForDns from './waitfordns.js'; import waitForDns from './waitfordns.js';
const debug = debugModule('box:dns/digitalocean'); const { log, trace } = logger('dns/digitalocean');
const DIGITALOCEAN_ENDPOINT = 'https://api.digitalocean.com'; const DIGITALOCEAN_ENDPOINT = 'https://api.digitalocean.com';
@@ -34,7 +34,7 @@ async function getZoneRecords(domainConfig, zoneName, name, type) {
let nextPage = null, matchingRecords = []; let nextPage = null, matchingRecords = [];
debug(`getInternal: getting dns records of ${zoneName} with ${name} and type ${type}`); log(`getInternal: getting dns records of ${zoneName} with ${name} and type ${type}`);
do { do {
const url = nextPage ? nextPage : DIGITALOCEAN_ENDPOINT + '/v2/domains/' + zoneName + '/records'; const url = nextPage ? nextPage : DIGITALOCEAN_ENDPOINT + '/v2/domains/' + zoneName + '/records';
@@ -70,7 +70,7 @@ async function upsert(domainObject, location, type, values) {
zoneName = domainObject.zoneName, zoneName = domainObject.zoneName,
name = dns.getName(domainObject, location, type) || '@'; name = dns.getName(domainObject, location, type) || '@';
debug('upsert: %s for zone %s of type %s with values %j', name, zoneName, type, values); log('upsert: %s for zone %s of type %s with values %j', name, zoneName, type, values);
const records = await getZoneRecords(domainConfig, zoneName, name, type); const records = await getZoneRecords(domainConfig, zoneName, name, type);
@@ -136,10 +136,10 @@ async function upsert(domainObject, location, type, values) {
.retry(5) .retry(5)
.ok(() => true)); .ok(() => true));
if (error) debug(`upsert: error removing record ${records[j].id}: ${error.message}`); if (error) log(`upsert: error removing record ${records[j].id}: ${error.message}`);
} }
debug('upsert: completed with recordIds:%j', recordIds); log('upsert: completed with recordIds:%j', recordIds);
} }
async function get(domainObject, location, type) { async function get(domainObject, location, type) {
@@ -229,17 +229,17 @@ async function verifyDomainConfig(domainObject) {
if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers'); if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers');
if (nameservers.map(function (n) { return n.toLowerCase(); }).indexOf('ns1.digitalocean.com') === -1) { if (nameservers.map(function (n) { return n.toLowerCase(); }).indexOf('ns1.digitalocean.com') === -1) {
debug('verifyDomainConfig: %j does not contains DO NS', nameservers); log('verifyDomainConfig: %j does not contains DO NS', nameservers);
if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to DigitalOcean'); if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to DigitalOcean');
} }
const location = 'cloudrontestdns'; const location = 'cloudrontestdns';
await upsert(domainObject, location, 'A', [ ip ]); await upsert(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record added'); log('verifyDomainConfig: Test A record added');
await del(domainObject, location, 'A', [ ip ]); await del(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record removed again'); log('verifyDomainConfig: Test A record removed again');
return credentials; return credentials;
} }

20
src/dns/dnsimple.js
View File

@@ -1,14 +1,14 @@
import assert from 'node:assert'; import assert from 'node:assert';
import BoxError from '../boxerror.js'; import BoxError from '../boxerror.js';
import constants from '../constants.js'; import constants from '../constants.js';
import debugModule from 'debug'; import logger from '../logger.js';
import dig from '../dig.js'; import dig from '../dig.js';
import dns from '../dns.js'; import dns from '../dns.js';
import safe from 'safetydance'; import safe from 'safetydance';
import superagent from '@cloudron/superagent'; import superagent from '@cloudron/superagent';
import waitForDns from './waitfordns.js'; import waitForDns from './waitfordns.js';
const debug = debugModule('box:dns/dnsimple'); const { log, trace } = logger('dns/dnsimple');
const DNSIMPLE_API = 'https://api.dnsimple.com/v2'; const DNSIMPLE_API = 'https://api.dnsimple.com/v2';
@@ -70,7 +70,7 @@ async function getDnsRecords(domainConfig, zoneName, name, type) {
assert.strictEqual(typeof name, 'string'); assert.strictEqual(typeof name, 'string');
assert.strictEqual(typeof type, 'string'); assert.strictEqual(typeof type, 'string');
debug(`get: ${name} in zone ${zoneName} of type ${type}`); log(`get: ${name} in zone ${zoneName} of type ${type}`);
const { accountId, zoneId } = await getZone(domainConfig, zoneName); const { accountId, zoneId } = await getZone(domainConfig, zoneName);
const [error, response] = await safe(superagent.get(`${DNSIMPLE_API}/${accountId}/zones/${zoneId}/records?name=${name}&type=${type}`) const [error, response] = await safe(superagent.get(`${DNSIMPLE_API}/${accountId}/zones/${zoneId}/records?name=${name}&type=${type}`)
@@ -96,7 +96,7 @@ async function upsert(domainObject, location, type, values) {
zoneName = domainObject.zoneName, zoneName = domainObject.zoneName,
name = dns.getName(domainObject, location, type) || ''; name = dns.getName(domainObject, location, type) || '';
debug(`upsert: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`); log(`upsert: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
const { accountId, zoneId } = await getZone(domainConfig, zoneName); const { accountId, zoneId } = await getZone(domainConfig, zoneName);
const records = await getDnsRecords(domainConfig, zoneName, name, type); const records = await getDnsRecords(domainConfig, zoneName, name, type);
@@ -157,10 +157,10 @@ async function upsert(domainObject, location, type, values) {
.timeout(30 * 1000) .timeout(30 * 1000)
.ok(() => true)); .ok(() => true));
if (error) debug(`upsert: error removing record ${records[j].id}: ${error.message}`); if (error) log(`upsert: error removing record ${records[j].id}: ${error.message}`);
} }
debug('upsert: completed with recordIds:%j', recordIds); log('upsert: completed with recordIds:%j', recordIds);
} }
async function get(domainObject, location, type) { async function get(domainObject, location, type) {
@@ -186,7 +186,7 @@ async function del(domainObject, location, type, values) {
zoneName = domainObject.zoneName, zoneName = domainObject.zoneName,
name = dns.getName(domainObject, location, type) || ''; name = dns.getName(domainObject, location, type) || '';
debug(`del: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`); log(`del: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
const { accountId, zoneId } = await getZone(domainConfig, zoneName); const { accountId, zoneId } = await getZone(domainConfig, zoneName);
const records = await getDnsRecords(domainConfig, zoneName, name, type); const records = await getDnsRecords(domainConfig, zoneName, name, type);
@@ -241,17 +241,17 @@ async function verifyDomainConfig(domainObject) {
if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers'); if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers');
if (!nameservers.every(function (n) { return n.toLowerCase().indexOf('dnsimple') !== -1; })) { // can be dnsimple.com or dnsimple-edge.org if (!nameservers.every(function (n) { return n.toLowerCase().indexOf('dnsimple') !== -1; })) { // can be dnsimple.com or dnsimple-edge.org
debug('verifyDomainConfig: %j does not contain dnsimple NS', nameservers); log('verifyDomainConfig: %j does not contain dnsimple NS', nameservers);
if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to dnsimple'); if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to dnsimple');
} }
const location = 'cloudrontestdns'; const location = 'cloudrontestdns';
await upsert(domainObject, location, 'A', [ ip ]); await upsert(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record added'); log('verifyDomainConfig: Test A record added');
await del(domainObject, location, 'A', [ ip ]); await del(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record removed again'); log('verifyDomainConfig: Test A record removed again');
return credentials; return credentials;
} }

16
src/dns/gandi.js
View File

@@ -1,14 +1,14 @@
import assert from 'node:assert'; import assert from 'node:assert';
import BoxError from '../boxerror.js'; import BoxError from '../boxerror.js';
import constants from '../constants.js'; import constants from '../constants.js';
import debugModule from 'debug'; import logger from '../logger.js';
import dig from '../dig.js'; import dig from '../dig.js';
import dns from '../dns.js'; import dns from '../dns.js';
import safe from 'safetydance'; import safe from 'safetydance';
import superagent from '@cloudron/superagent'; import superagent from '@cloudron/superagent';
import waitForDns from './waitfordns.js'; import waitForDns from './waitfordns.js';
const debug = debugModule('box:dns/gandi'); const { log, trace } = logger('dns/gandi');
const GANDI_API = 'https://dns.api.gandi.net/api/v5'; const GANDI_API = 'https://dns.api.gandi.net/api/v5';
@@ -54,7 +54,7 @@ async function upsert(domainObject, location, type, values) {
zoneName = domainObject.zoneName, zoneName = domainObject.zoneName,
name = dns.getName(domainObject, location, type) || '@'; name = dns.getName(domainObject, location, type) || '@';
debug(`upsert: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`); log(`upsert: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
const data = { const data = {
'rrset_ttl': 300, // this is the minimum allowed 'rrset_ttl': 300, // this is the minimum allowed
@@ -79,7 +79,7 @@ async function get(domainObject, location, type) {
zoneName = domainObject.zoneName, zoneName = domainObject.zoneName,
name = dns.getName(domainObject, location, type) || '@'; name = dns.getName(domainObject, location, type) || '@';
debug(`get: ${name} in zone ${zoneName} of type ${type}`); log(`get: ${name} in zone ${zoneName} of type ${type}`);
const [error, response] = await safe(createRequest('GET', `${GANDI_API}/domains/${zoneName}/records/${name}/${type}`, domainConfig)); const [error, response] = await safe(createRequest('GET', `${GANDI_API}/domains/${zoneName}/records/${name}/${type}`, domainConfig));
@@ -101,7 +101,7 @@ async function del(domainObject, location, type, values) {
zoneName = domainObject.zoneName, zoneName = domainObject.zoneName,
name = dns.getName(domainObject, location, type) || '@'; name = dns.getName(domainObject, location, type) || '@';
debug(`del: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`); log(`del: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
const [error, response] = await safe(createRequest('DELETE', `${GANDI_API}/domains/${zoneName}/records/${name}/${type}`, domainConfig)); const [error, response] = await safe(createRequest('DELETE', `${GANDI_API}/domains/${zoneName}/records/${name}/${type}`, domainConfig));
@@ -148,17 +148,17 @@ async function verifyDomainConfig(domainObject) {
if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers'); if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers');
if (!nameservers.every(function (n) { return n.toLowerCase().indexOf('.gandi.net') !== -1; })) { if (!nameservers.every(function (n) { return n.toLowerCase().indexOf('.gandi.net') !== -1; })) {
debug('verifyDomainConfig: %j does not contain Gandi NS', nameservers); log('verifyDomainConfig: %j does not contain Gandi NS', nameservers);
if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Gandi'); if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Gandi');
} }
const location = 'cloudrontestdns'; const location = 'cloudrontestdns';
await upsert(domainObject, location, 'A', [ ip ]); await upsert(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record added'); log('verifyDomainConfig: Test A record added');
await del(domainObject, location, 'A', [ ip ]); await del(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record removed again'); log('verifyDomainConfig: Test A record removed again');
return credentials; return credentials;
} }

12
src/dns/gcdns.js
View File

@@ -1,7 +1,7 @@
import assert from 'node:assert'; import assert from 'node:assert';
import BoxError from '../boxerror.js'; import BoxError from '../boxerror.js';
import constants from '../constants.js'; import constants from '../constants.js';
import debugModule from 'debug'; import logger from '../logger.js';
import dig from '../dig.js'; import dig from '../dig.js';
import dns from '../dns.js'; import dns from '../dns.js';
import { DNS as GCDNS } from '@google-cloud/dns'; import { DNS as GCDNS } from '@google-cloud/dns';
@@ -9,7 +9,7 @@ import safe from 'safetydance';
import waitForDns from './waitfordns.js'; import waitForDns from './waitfordns.js';
import _ from '../underscore.js'; import _ from '../underscore.js';
const debug = debugModule('box:dns/gcdns'); const { log, trace } = logger('dns/gcdns');
function removePrivateFields(domainObject) { function removePrivateFields(domainObject) {
@@ -66,7 +66,7 @@ async function upsert(domainObject, location, type, values) {
zoneName = domainObject.zoneName, zoneName = domainObject.zoneName,
fqdn = dns.fqdn(location, domainObject.domain); fqdn = dns.fqdn(location, domainObject.domain);
debug('add: %s for zone %s of type %s with values %j', fqdn, zoneName, type, values); log('add: %s for zone %s of type %s with values %j', fqdn, zoneName, type, values);
const zone = await getZoneByName(getDnsCredentials(domainConfig), zoneName); const zone = await getZoneByName(getDnsCredentials(domainConfig), zoneName);
@@ -170,16 +170,16 @@ async function verifyDomainConfig(domainObject) {
const definedNS = zone.metadata.nameServers.map(function(r) { return r.replace(/\.$/, ''); }); const definedNS = zone.metadata.nameServers.map(function(r) { return r.replace(/\.$/, ''); });
if (!_.isEqual(definedNS.sort(), nameservers.sort())) { if (!_.isEqual(definedNS.sort(), nameservers.sort())) {
debug('verifyDomainConfig: %j and %j do not match', nameservers, definedNS); log('verifyDomainConfig: %j and %j do not match', nameservers, definedNS);
if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Google Cloud DNS'); if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Google Cloud DNS');
} }
const location = 'cloudrontestdns'; const location = 'cloudrontestdns';
await upsert(domainObject, location, 'A', [ ip ]); await upsert(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record added'); log('verifyDomainConfig: Test A record added');
await del(domainObject, location, 'A', [ ip ]); await del(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record removed again'); log('verifyDomainConfig: Test A record removed again');
return credentials; return credentials;
} }

16
src/dns/godaddy.js
View File

@@ -1,14 +1,14 @@
import assert from 'node:assert'; import assert from 'node:assert';
import BoxError from '../boxerror.js'; import BoxError from '../boxerror.js';
import constants from '../constants.js'; import constants from '../constants.js';
import debugModule from 'debug'; import logger from '../logger.js';
import dig from '../dig.js'; import dig from '../dig.js';
import dns from '../dns.js'; import dns from '../dns.js';
import safe from 'safetydance'; import safe from 'safetydance';
import superagent from '@cloudron/superagent'; import superagent from '@cloudron/superagent';
import waitForDns from './waitfordns.js'; import waitForDns from './waitfordns.js';
const debug = debugModule('box:dns/godaddy'); const { log, trace } = logger('dns/godaddy');
// const GODADDY_API_OTE = 'https://api.ote-godaddy.com/v1/domains'; // const GODADDY_API_OTE = 'https://api.ote-godaddy.com/v1/domains';
@@ -37,7 +37,7 @@ async function upsert(domainObject, location, type, values) {
zoneName = domainObject.zoneName, zoneName = domainObject.zoneName,
name = dns.getName(domainObject, location, type) || '@'; name = dns.getName(domainObject, location, type) || '@';
debug(`upsert: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`); log(`upsert: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
const records = []; const records = [];
for (const value of values) { for (const value of values) {
@@ -75,7 +75,7 @@ async function get(domainObject, location, type) {
zoneName = domainObject.zoneName, zoneName = domainObject.zoneName,
name = dns.getName(domainObject, location, type) || '@'; name = dns.getName(domainObject, location, type) || '@';
debug(`get: ${name} in zone ${zoneName} of type ${type}`); log(`get: ${name} in zone ${zoneName} of type ${type}`);
const [error, response] = await safe(superagent.get(`${GODADDY_API}/${zoneName}/records/${type}/${name}`) const [error, response] = await safe(superagent.get(`${GODADDY_API}/${zoneName}/records/${type}/${name}`)
.set('Authorization', `sso-key ${domainConfig.apiKey}:${domainConfig.apiSecret}`) .set('Authorization', `sso-key ${domainConfig.apiKey}:${domainConfig.apiSecret}`)
@@ -114,7 +114,7 @@ async function del(domainObject, location, type, values) {
zoneName = domainObject.zoneName, zoneName = domainObject.zoneName,
name = dns.getName(domainObject, location, type) || '@'; name = dns.getName(domainObject, location, type) || '@';
debug(`del: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`); log(`del: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
const result = await get(domainObject, location, type); const result = await get(domainObject, location, type);
if (result.length === 0) return; if (result.length === 0) return;
@@ -171,17 +171,17 @@ async function verifyDomainConfig(domainObject) {
if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers'); if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers');
if (!nameservers.every(function (n) { return n.toLowerCase().indexOf('.domaincontrol.com') !== -1 || n.toLowerCase().indexOf('.secureserver.net') !== -1; })) { if (!nameservers.every(function (n) { return n.toLowerCase().indexOf('.domaincontrol.com') !== -1 || n.toLowerCase().indexOf('.secureserver.net') !== -1; })) {
debug('verifyDomainConfig: %j does not contain GoDaddy NS', nameservers); log('verifyDomainConfig: %j does not contain GoDaddy NS', nameservers);
if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to GoDaddy'); if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to GoDaddy');
} }
const location = 'cloudrontestdns'; const location = 'cloudrontestdns';
await upsert(domainObject, location, 'A', [ ip ]); await upsert(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record added'); log('verifyDomainConfig: Test A record added');
await del(domainObject, location, 'A', [ ip ]); await del(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record removed again'); log('verifyDomainConfig: Test A record removed again');
return credentials; return credentials;
} }

18
src/dns/hetzner.js
View File

@@ -1,14 +1,14 @@
import assert from 'node:assert'; import assert from 'node:assert';
import BoxError from '../boxerror.js'; import BoxError from '../boxerror.js';
import constants from '../constants.js'; import constants from '../constants.js';
import debugModule from 'debug'; import logger from '../logger.js';
import dig from '../dig.js'; import dig from '../dig.js';
import dns from '../dns.js'; import dns from '../dns.js';
import safe from 'safetydance'; import safe from 'safetydance';
import superagent from '@cloudron/superagent'; import superagent from '@cloudron/superagent';
import waitForDns from './waitfordns.js'; import waitForDns from './waitfordns.js';
const debug = debugModule('box:dns/hetzner'); const { log, trace } = logger('dns/hetzner');
const ENDPOINT = 'https://dns.hetzner.com/api/v1'; const ENDPOINT = 'https://dns.hetzner.com/api/v1';
@@ -55,7 +55,7 @@ async function getZoneRecords(domainConfig, zone, name, type) {
let page = 1, matchingRecords = []; let page = 1, matchingRecords = [];
debug(`getZoneRecords: getting dns records of ${zone.name} with ${name} and type ${type}`); log(`getZoneRecords: getting dns records of ${zone.name} with ${name} and type ${type}`);
const perPage = 50; const perPage = 50;
@@ -94,7 +94,7 @@ async function upsert(domainObject, location, type, values) {
zoneName = domainObject.zoneName, zoneName = domainObject.zoneName,
name = dns.getName(domainObject, location, type) || '@'; name = dns.getName(domainObject, location, type) || '@';
debug(`upsert: ${name} for zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`); log(`upsert: ${name} for zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
const zone = await getZone(domainConfig, zoneName); const zone = await getZone(domainConfig, zoneName);
const records = await getZoneRecords(domainConfig, zone, name, type); const records = await getZoneRecords(domainConfig, zone, name, type);
@@ -147,10 +147,10 @@ async function upsert(domainObject, location, type, values) {
.retry(5) .retry(5)
.ok(() => true)); .ok(() => true));
if (error) debug(`upsert: error removing record ${records[j].id}: ${error.message}`); if (error) log(`upsert: error removing record ${records[j].id}: ${error.message}`);
} }
debug('upsert: completed'); log('upsert: completed');
} }
async function get(domainObject, location, type) { async function get(domainObject, location, type) {
@@ -235,17 +235,17 @@ async function verifyDomainConfig(domainObject) {
// https://docs.hetzner.com/dns-console/dns/general/dns-overview#the-hetzner-online-name-servers-are // https://docs.hetzner.com/dns-console/dns/general/dns-overview#the-hetzner-online-name-servers-are
if (!nameservers.every(function (n) { return n.toLowerCase().search(/hetzner|your-server|second-ns/) !== -1; })) { if (!nameservers.every(function (n) { return n.toLowerCase().search(/hetzner|your-server|second-ns/) !== -1; })) {
debug('verifyDomainConfig: %j does not contain Hetzner NS', nameservers); log('verifyDomainConfig: %j does not contain Hetzner NS', nameservers);
if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Hetzner'); if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Hetzner');
} }
const location = 'cloudrontestdns'; const location = 'cloudrontestdns';
await upsert(domainObject, location, 'A', [ ip ]); await upsert(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record added'); log('verifyDomainConfig: Test A record added');
await del(domainObject, location, 'A', [ ip ]); await del(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record removed again'); log('verifyDomainConfig: Test A record removed again');
return credentials; return credentials;
} }

16
src/dns/hetznercloud.js
View File

@@ -1,7 +1,7 @@
import assert from 'node:assert'; import assert from 'node:assert';
import BoxError from '../boxerror.js'; import BoxError from '../boxerror.js';
import constants from '../constants.js'; import constants from '../constants.js';
import debugModule from 'debug'; import logger from '../logger.js';
import dig from '../dig.js'; import dig from '../dig.js';
import dns from '../dns.js'; import dns from '../dns.js';
import promiseRetry from '../promise-retry.js'; import promiseRetry from '../promise-retry.js';
@@ -9,7 +9,7 @@ import safe from 'safetydance';
import superagent from '@cloudron/superagent'; import superagent from '@cloudron/superagent';
import waitForDns from './waitfordns.js'; import waitForDns from './waitfordns.js';
const debug = debugModule('box:dns/hetznercloud'); const { log, trace } = logger('dns/hetznercloud');
// https://docs.hetzner.cloud/reference/cloud // https://docs.hetzner.cloud/reference/cloud
@@ -68,7 +68,7 @@ async function getRecords(domainConfig, zone, name, type) {
assert.strictEqual(typeof name, 'string'); assert.strictEqual(typeof name, 'string');
assert.strictEqual(typeof type, 'string'); assert.strictEqual(typeof type, 'string');
debug(`getRecords: getting dns records of ${zone.name} with ${name} and type ${type}`); log(`getRecords: getting dns records of ${zone.name} with ${name} and type ${type}`);
const [error, response] = await safe(superagent.get(`${ENDPOINT}/zones/${zone.id}/rrsets/${name}/${type.toUpperCase()}`) const [error, response] = await safe(superagent.get(`${ENDPOINT}/zones/${zone.id}/rrsets/${name}/${type.toUpperCase()}`)
.set('Authorization', `Bearer ${domainConfig.token}`) .set('Authorization', `Bearer ${domainConfig.token}`)
@@ -88,7 +88,7 @@ async function waitForAction(domainConfig, id) {
assert.strictEqual(typeof domainConfig, 'object'); assert.strictEqual(typeof domainConfig, 'object');
assert.strictEqual(typeof id, 'number'); assert.strictEqual(typeof id, 'number');
await promiseRetry({ times: 100, interval: 1000, debug }, async () => { await promiseRetry({ times: 100, interval: 1000, debug: log }, async () => {
const [error, response] = await safe(superagent.get(`${ENDPOINT}/actions/${id}`) const [error, response] = await safe(superagent.get(`${ENDPOINT}/actions/${id}`)
.set('Authorization', `Bearer ${domainConfig.token}`) .set('Authorization', `Bearer ${domainConfig.token}`)
.timeout(30 * 1000) .timeout(30 * 1000)
@@ -141,7 +141,7 @@ async function upsert(domainObject, location, type, values) {
zoneName = domainObject.zoneName, zoneName = domainObject.zoneName,
name = dns.getName(domainObject, location, type) || '@'; name = dns.getName(domainObject, location, type) || '@';
debug(`upsert: ${name} for zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`); log(`upsert: ${name} for zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
const zone = await getZone(domainConfig, zoneName); const zone = await getZone(domainConfig, zoneName);
const records = await getRecords(domainConfig, zone, name, type); const records = await getRecords(domainConfig, zone, name, type);
@@ -213,17 +213,17 @@ async function verifyDomainConfig(domainObject) {
// https://docs.hetzner.com/dns-console/dns/general/dns-overview#the-hetzner-online-name-servers-are // https://docs.hetzner.com/dns-console/dns/general/dns-overview#the-hetzner-online-name-servers-are
if (!nameservers.every(function (n) { return n.toLowerCase().search(/hetzner|your-server|second-ns/) !== -1; })) { if (!nameservers.every(function (n) { return n.toLowerCase().search(/hetzner|your-server|second-ns/) !== -1; })) {
debug('verifyDomainConfig: %j does not contain Hetzner NS', nameservers); log('verifyDomainConfig: %j does not contain Hetzner NS', nameservers);
if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Hetzner'); if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Hetzner');
} }
const location = 'cloudrontestdns'; const location = 'cloudrontestdns';
await upsert(domainObject, location, 'A', [ ip ]); await upsert(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record added'); log('verifyDomainConfig: Test A record added');
await del(domainObject, location, 'A', [ ip ]); await del(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record removed again'); log('verifyDomainConfig: Test A record removed again');
return credentials; return credentials;
} }

18
src/dns/inwx.js
View File

@@ -2,13 +2,13 @@ import { ApiClient, Language } from 'domrobot-client';
import assert from 'node:assert'; import assert from 'node:assert';
import BoxError from '../boxerror.js'; import BoxError from '../boxerror.js';
import constants from '../constants.js'; import constants from '../constants.js';
import debugModule from 'debug'; import logger from '../logger.js';
import dig from '../dig.js'; import dig from '../dig.js';
import dns from '../dns.js'; import dns from '../dns.js';
import safe from 'safetydance'; import safe from 'safetydance';
import waitForDns from './waitfordns.js'; import waitForDns from './waitfordns.js';
const debug = debugModule('box:dns/inwx'); const { log, trace } = logger('dns/inwx');
function formatError(response) { function formatError(response) {
@@ -49,7 +49,7 @@ async function getDnsRecords(domainConfig, apiClient, zoneName, fqdn, type) {
assert.strictEqual(typeof fqdn, 'string'); assert.strictEqual(typeof fqdn, 'string');
assert.strictEqual(typeof type, 'string'); assert.strictEqual(typeof type, 'string');
debug(`getDnsRecords: ${fqdn} in zone ${zoneName} of type ${type}`); log(`getDnsRecords: ${fqdn} in zone ${zoneName} of type ${type}`);
const [error, response] = await safe(apiClient.callApi('nameserver.info', { domain: zoneName, name: fqdn, type })); const [error, response] = await safe(apiClient.callApi('nameserver.info', { domain: zoneName, name: fqdn, type }));
if (error) throw new BoxError(BoxError.NETWORK_ERROR, error); if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
@@ -67,7 +67,7 @@ async function upsert(domainObject, location, type, values) {
const domainConfig = domainObject.config, const domainConfig = domainObject.config,
zoneName = domainObject.zoneName; zoneName = domainObject.zoneName;
debug(`upsert: ${location} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`); log(`upsert: ${location} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
const apiClient = await login(domainConfig); const apiClient = await login(domainConfig);
const fqdn = dns.fqdn(location, domainObject.domain); const fqdn = dns.fqdn(location, domainObject.domain);
@@ -140,7 +140,7 @@ async function del(domainObject, location, type, values) {
const domainConfig = domainObject.config, const domainConfig = domainObject.config,
zoneName = domainObject.zoneName; zoneName = domainObject.zoneName;
debug(`del: ${location} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`); log(`del: ${location} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
const apiClient = await login(domainConfig); const apiClient = await login(domainConfig);
const fqdn = dns.fqdn(location, domainObject.domain); const fqdn = dns.fqdn(location, domainObject.domain);
@@ -148,7 +148,7 @@ async function del(domainObject, location, type, values) {
if (result.length === 0) return; if (result.length === 0) return;
const tmp = result.filter(function (record) { return values.some(function (value) { return value === record.content; }); }); const tmp = result.filter(function (record) { return values.some(function (value) { return value === record.content; }); });
debug('del: %j', tmp); log('del: %j', tmp);
for (const r of tmp) { for (const r of tmp) {
const [error, response] = await safe(apiClient.callApi('nameserver.deleteRecord', { id: r.id })); const [error, response] = await safe(apiClient.callApi('nameserver.deleteRecord', { id: r.id }));
@@ -194,17 +194,17 @@ async function verifyDomainConfig(domainObject) {
if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers'); if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers');
if (!nameservers.every(function (n) { return n.toLowerCase().search(/inwx|xnameserver|domrobot/) !== -1; })) { if (!nameservers.every(function (n) { return n.toLowerCase().search(/inwx|xnameserver|domrobot/) !== -1; })) {
debug('verifyDomainConfig: %j does not contain INWX NS', nameservers); log('verifyDomainConfig: %j does not contain INWX NS', nameservers);
if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to INWX'); if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to INWX');
} }
const location = 'cloudrontestdns'; const location = 'cloudrontestdns';
await upsert(domainObject, location, 'A', [ ip ]); await upsert(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record added'); log('verifyDomainConfig: Test A record added');
await del(domainObject, location, 'A', [ ip ]); await del(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record removed again'); log('verifyDomainConfig: Test A record removed again');
return credentials; return credentials;
} }

18
src/dns/linode.js
View File

@@ -1,14 +1,14 @@
import assert from 'node:assert'; import assert from 'node:assert';
import constants from '../constants.js'; import constants from '../constants.js';
import BoxError from '../boxerror.js'; import BoxError from '../boxerror.js';
import debugModule from 'debug'; import logger from '../logger.js';
import dig from '../dig.js'; import dig from '../dig.js';
import dns from '../dns.js'; import dns from '../dns.js';
import safe from 'safetydance'; import safe from 'safetydance';
import superagent from '@cloudron/superagent'; import superagent from '@cloudron/superagent';
import waitForDns from './waitfordns.js'; import waitForDns from './waitfordns.js';
const debug = debugModule('box:dns/linode'); const { log, trace } = logger('dns/linode');
const LINODE_ENDPOINT = 'https://api.linode.com/v4'; const LINODE_ENDPOINT = 'https://api.linode.com/v4';
@@ -47,7 +47,7 @@ async function getZoneId(domainConfig, zoneName) {
if (!zone || !zone.id) throw new BoxError(BoxError.NOT_FOUND, 'Zone not found'); if (!zone || !zone.id) throw new BoxError(BoxError.NOT_FOUND, 'Zone not found');
debug(`getZoneId: zone id of ${zoneName} is ${zone.id}`); log(`getZoneId: zone id of ${zoneName} is ${zone.id}`);
return zone.id; return zone.id;
} }
@@ -58,7 +58,7 @@ async function getZoneRecords(domainConfig, zoneName, name, type) {
assert.strictEqual(typeof name, 'string'); assert.strictEqual(typeof name, 'string');
assert.strictEqual(typeof type, 'string'); assert.strictEqual(typeof type, 'string');
debug(`getInternal: getting dns records of ${zoneName} with ${name} and type ${type}`); log(`getInternal: getting dns records of ${zoneName} with ${name} and type ${type}`);
const zoneId = await getZoneId(domainConfig, zoneName); const zoneId = await getZoneId(domainConfig, zoneName);
@@ -113,7 +113,7 @@ async function upsert(domainObject, location, type, values) {
zoneName = domainObject.zoneName, zoneName = domainObject.zoneName,
name = dns.getName(domainObject, location, type) || ''; name = dns.getName(domainObject, location, type) || '';
debug('upsert: %s for zone %s of type %s with values %j', name, zoneName, type, values); log('upsert: %s for zone %s of type %s with values %j', name, zoneName, type, values);
const { zoneId, records } = await getZoneRecords(domainConfig, zoneName, name, type); const { zoneId, records } = await getZoneRecords(domainConfig, zoneName, name, type);
let i = 0; // used to track available records to update instead of create let i = 0; // used to track available records to update instead of create
@@ -176,7 +176,7 @@ async function upsert(domainObject, location, type, values) {
.retry(5) .retry(5)
.ok(() => true)); .ok(() => true));
if (error) debug(`upsert: error removing record ${records[j].id}: ${error.message}`); if (error) log(`upsert: error removing record ${records[j].id}: ${error.message}`);
} }
} }
@@ -244,17 +244,17 @@ async function verifyDomainConfig(domainObject) {
if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers'); if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers');
if (nameservers.map(function (n) { return n.toLowerCase(); }).indexOf('ns1.linode.com') === -1) { if (nameservers.map(function (n) { return n.toLowerCase(); }).indexOf('ns1.linode.com') === -1) {
debug('verifyDomainConfig: %j does not contains linode NS', nameservers); log('verifyDomainConfig: %j does not contains linode NS', nameservers);
if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Linode'); if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Linode');
} }
const location = 'cloudrontestdns'; const location = 'cloudrontestdns';
await upsert(domainObject, location, 'A', [ ip ]); await upsert(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record added'); log('verifyDomainConfig: Test A record added');
await del(domainObject, location, 'A', [ ip ]); await del(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record removed again'); log('verifyDomainConfig: Test A record removed again');
return credentials; return credentials;
} }

6
src/dns/manual.js
View File

@@ -1,12 +1,12 @@
import assert from 'node:assert'; import assert from 'node:assert';
import BoxError from '../boxerror.js'; import BoxError from '../boxerror.js';
import debugModule from 'debug'; import logger from '../logger.js';
import dig from '../dig.js'; import dig from '../dig.js';
import dns from '../dns.js'; import dns from '../dns.js';
import safe from 'safetydance'; import safe from 'safetydance';
import waitForDns from './waitfordns.js'; import waitForDns from './waitfordns.js';
const debug = debugModule('box:dns/manual'); const { log, trace } = logger('dns/manual');
function removePrivateFields(domainObject) { function removePrivateFields(domainObject) {
@@ -24,7 +24,7 @@ async function upsert(domainObject, location, type, values) {
assert.strictEqual(typeof type, 'string'); assert.strictEqual(typeof type, 'string');
assert(Array.isArray(values)); assert(Array.isArray(values));
debug('upsert: %s for zone %s of type %s with values %j', location, domainObject.zoneName, type, values); log('upsert: %s for zone %s of type %s with values %j', location, domainObject.zoneName, type, values);
return; return;
} }

14
src/dns/namecheap.js
View File

@@ -1,7 +1,7 @@
import assert from 'node:assert'; import assert from 'node:assert';
import BoxError from '../boxerror.js'; import BoxError from '../boxerror.js';
import constants from '../constants.js'; import constants from '../constants.js';
import debugModule from 'debug'; import logger from '../logger.js';
import dig from '../dig.js'; import dig from '../dig.js';
import dns from '../dns.js'; import dns from '../dns.js';
import network from '../network.js'; import network from '../network.js';
@@ -12,7 +12,7 @@ import util from 'node:util';
import waitForDns from './waitfordns.js'; import waitForDns from './waitfordns.js';
import xml2js from 'xml2js'; import xml2js from 'xml2js';
const debug = debugModule('box:dns/namecheap'); const { log, trace } = logger('dns/namecheap');
const ENDPOINT = 'https://api.namecheap.com/xml.response'; const ENDPOINT = 'https://api.namecheap.com/xml.response';
@@ -130,7 +130,7 @@ async function upsert(domainObject, subdomain, type, values) {
subdomain = dns.getName(domainObject, subdomain, type) || '@'; subdomain = dns.getName(domainObject, subdomain, type) || '@';
debug('upsert: %s for zone %s of type %s with values %j', subdomain, zoneName, type, values); log('upsert: %s for zone %s of type %s with values %j', subdomain, zoneName, type, values);
const result = await getZone(domainConfig, zoneName); const result = await getZone(domainConfig, zoneName);
@@ -210,7 +210,7 @@ async function del(domainObject, subdomain, type, values) {
subdomain = dns.getName(domainObject, subdomain, type) || '@'; subdomain = dns.getName(domainObject, subdomain, type) || '@';
debug('del: %s for zone %s of type %s with values %j', subdomain, zoneName, type, values); log('del: %s for zone %s of type %s with values %j', subdomain, zoneName, type, values);
let result = await getZone(domainConfig, zoneName); let result = await getZone(domainConfig, zoneName);
if (result.length === 0) return; if (result.length === 0) return;
@@ -261,17 +261,17 @@ async function verifyDomainConfig(domainObject) {
if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers'); if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers');
if (nameservers.some(function (n) { return n.toLowerCase().indexOf('.registrar-servers.com') === -1; })) { if (nameservers.some(function (n) { return n.toLowerCase().indexOf('.registrar-servers.com') === -1; })) {
debug('verifyDomainConfig: %j does not contains NC NS', nameservers); log('verifyDomainConfig: %j does not contains NC NS', nameservers);
if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to NameCheap'); if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to NameCheap');
} }
const testSubdomain = 'cloudrontestdns'; const testSubdomain = 'cloudrontestdns';
await upsert(domainObject, testSubdomain, 'A', [ip]); await upsert(domainObject, testSubdomain, 'A', [ip]);
debug('verifyDomainConfig: Test A record added'); log('verifyDomainConfig: Test A record added');
await del(domainObject, testSubdomain, 'A', [ip]); await del(domainObject, testSubdomain, 'A', [ip]);
debug('verifyDomainConfig: Test A record removed again'); log('verifyDomainConfig: Test A record removed again');
return credentials; return credentials;
} }

20
src/dns/namecom.js
View File

@@ -1,14 +1,14 @@
import assert from 'node:assert'; import assert from 'node:assert';
import BoxError from '../boxerror.js'; import BoxError from '../boxerror.js';
import constants from '../constants.js'; import constants from '../constants.js';
import debugModule from 'debug'; import logger from '../logger.js';
import dig from '../dig.js'; import dig from '../dig.js';
import dns from '../dns.js'; import dns from '../dns.js';
import safe from 'safetydance'; import safe from 'safetydance';
import superagent from '@cloudron/superagent'; import superagent from '@cloudron/superagent';
import waitForDns from './waitfordns.js'; import waitForDns from './waitfordns.js';
const debug = debugModule('box:dns/namecom'); const { log, trace } = logger('dns/namecom');
const NAMECOM_API = 'https://api.name.com/v4'; const NAMECOM_API = 'https://api.name.com/v4';
@@ -33,7 +33,7 @@ async function addRecord(domainConfig, zoneName, name, type, values) {
assert.strictEqual(typeof type, 'string'); assert.strictEqual(typeof type, 'string');
assert(Array.isArray(values)); assert(Array.isArray(values));
debug(`add: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`); log(`add: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
const data = { const data = {
host: name, host: name,
@@ -71,7 +71,7 @@ async function updateRecord(domainConfig, zoneName, recordId, name, type, values
assert.strictEqual(typeof type, 'string'); assert.strictEqual(typeof type, 'string');
assert(Array.isArray(values)); assert(Array.isArray(values));
debug(`update:${recordId} on ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`); log(`update:${recordId} on ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
const data = { const data = {
host: name, host: name,
@@ -107,7 +107,7 @@ async function getInternal(domainConfig, zoneName, name, type) {
assert.strictEqual(typeof name, 'string'); assert.strictEqual(typeof name, 'string');
assert.strictEqual(typeof type, 'string'); assert.strictEqual(typeof type, 'string');
debug(`getInternal: ${name} in zone ${zoneName} of type ${type}`); log(`getInternal: ${name} in zone ${zoneName} of type ${type}`);
const [error, response] = await safe(superagent.get(`${NAMECOM_API}/domains/${zoneName}/records`) const [error, response] = await safe(superagent.get(`${NAMECOM_API}/domains/${zoneName}/records`)
.auth(domainConfig.username, domainConfig.token) .auth(domainConfig.username, domainConfig.token)
@@ -144,7 +144,7 @@ async function upsert(domainObject, location, type, values) {
zoneName = domainObject.zoneName, zoneName = domainObject.zoneName,
name = dns.getName(domainObject, location, type) || ''; name = dns.getName(domainObject, location, type) || '';
debug(`upsert: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`); log(`upsert: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
const result = await getInternal(domainConfig, zoneName, name, type); const result = await getInternal(domainConfig, zoneName, name, type);
if (result.length === 0) return await addRecord(domainConfig, zoneName, name, type, values); if (result.length === 0) return await addRecord(domainConfig, zoneName, name, type, values);
@@ -176,7 +176,7 @@ async function del(domainObject, location, type, values) {
zoneName = domainObject.zoneName, zoneName = domainObject.zoneName,
name = dns.getName(domainObject, location, type) || ''; name = dns.getName(domainObject, location, type) || '';
debug(`del: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`); log(`del: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
const result = await getInternal(domainConfig, zoneName, name, type); const result = await getInternal(domainConfig, zoneName, name, type);
if (result.length === 0) return; if (result.length === 0) return;
@@ -227,17 +227,17 @@ async function verifyDomainConfig(domainObject) {
if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers'); if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers');
if (!nameservers.every(function (n) { return n.toLowerCase().indexOf('.name.com') !== -1; })) { if (!nameservers.every(function (n) { return n.toLowerCase().indexOf('.name.com') !== -1; })) {
debug('verifyDomainConfig: %j does not contain Name.com NS', nameservers); log('verifyDomainConfig: %j does not contain Name.com NS', nameservers);
if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to name.com'); if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to name.com');
} }
const location = 'cloudrontestdns'; const location = 'cloudrontestdns';
await upsert(domainObject, location, 'A', [ ip ]); await upsert(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record added'); log('verifyDomainConfig: Test A record added');
await del(domainObject, location, 'A', [ ip ]); await del(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record removed again'); log('verifyDomainConfig: Test A record removed again');
return credentials; return credentials;
} }

18
src/dns/netcup.js
View File

@@ -1,14 +1,14 @@
import assert from 'node:assert'; import assert from 'node:assert';
import BoxError from '../boxerror.js'; import BoxError from '../boxerror.js';
import constants from '../constants.js'; import constants from '../constants.js';
import debugModule from 'debug'; import logger from '../logger.js';
import dig from '../dig.js'; import dig from '../dig.js';
import dns from '../dns.js'; import dns from '../dns.js';
import safe from 'safetydance'; import safe from 'safetydance';
import superagent from '@cloudron/superagent'; import superagent from '@cloudron/superagent';
import waitForDns from './waitfordns.js'; import waitForDns from './waitfordns.js';
const debug = debugModule('box:dns/netcup'); const { log, trace } = logger('dns/netcup');
const API_ENDPOINT = 'https://ccp.netcup.net/run/webservice/servers/endpoint.php?JSON'; const API_ENDPOINT = 'https://ccp.netcup.net/run/webservice/servers/endpoint.php?JSON';
@@ -53,7 +53,7 @@ async function getAllRecords(domainConfig, apiSessionId, zoneName) {
assert.strictEqual(typeof apiSessionId, 'string'); assert.strictEqual(typeof apiSessionId, 'string');
assert.strictEqual(typeof zoneName, 'string'); assert.strictEqual(typeof zoneName, 'string');
debug(`getAllRecords: getting dns records of ${zoneName}`); log(`getAllRecords: getting dns records of ${zoneName}`);
const data = { const data = {
action: 'infoDnsRecords', action: 'infoDnsRecords',
@@ -82,7 +82,7 @@ async function upsert(domainObject, location, type, values) {
zoneName = domainObject.zoneName, zoneName = domainObject.zoneName,
name = dns.getName(domainObject, location, type) || '@'; name = dns.getName(domainObject, location, type) || '@';
debug('upsert: %s for zone %s of type %s with values %j', name, zoneName, type, values); log('upsert: %s for zone %s of type %s with values %j', name, zoneName, type, values);
const apiSessionId = await login(domainConfig); const apiSessionId = await login(domainConfig);
@@ -138,7 +138,7 @@ async function get(domainObject, location, type) {
zoneName = domainObject.zoneName, zoneName = domainObject.zoneName,
name = dns.getName(domainObject, location, type) || '@'; name = dns.getName(domainObject, location, type) || '@';
debug('get: %s for zone %s of type %s', name, zoneName, type); log('get: %s for zone %s of type %s', name, zoneName, type);
const apiSessionId = await login(domainConfig); const apiSessionId = await login(domainConfig);
@@ -158,7 +158,7 @@ async function del(domainObject, location, type, values) {
zoneName = domainObject.zoneName, zoneName = domainObject.zoneName,
name = dns.getName(domainObject, location, type) || '@'; name = dns.getName(domainObject, location, type) || '@';
debug('del: %s for zone %s of type %s with values %j', name, zoneName, type, values); log('del: %s for zone %s of type %s with values %j', name, zoneName, type, values);
const apiSessionId = await login(domainConfig); const apiSessionId = await login(domainConfig);
@@ -239,17 +239,17 @@ async function verifyDomainConfig(domainObject) {
if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers'); if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers');
if (!nameservers.every(function (n) { return n.toLowerCase().indexOf('dns.netcup.net') !== -1; })) { if (!nameservers.every(function (n) { return n.toLowerCase().indexOf('dns.netcup.net') !== -1; })) {
debug('verifyDomainConfig: %j does not contains Netcup NS', nameservers); log('verifyDomainConfig: %j does not contains Netcup NS', nameservers);
if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Netcup'); if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Netcup');
} }
const location = 'cloudrontestdns'; const location = 'cloudrontestdns';
await upsert(domainObject, location, 'A', [ ip ]); await upsert(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record added'); log('verifyDomainConfig: Test A record added');
await del(domainObject, location, 'A', [ ip ]); await del(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record removed again'); log('verifyDomainConfig: Test A record removed again');
return credentials; return credentials;
} }

6
src/dns/noop.js
View File

@@ -1,7 +1,7 @@
import assert from 'node:assert'; import assert from 'node:assert';
import debugModule from 'debug'; import logger from '../logger.js';
const debug = debugModule('box:dns/noop'); const { log, trace } = logger('dns/noop');
function removePrivateFields(domainObject) { function removePrivateFields(domainObject) {
@@ -18,7 +18,7 @@ async function upsert(domainObject, location, type, values) {
assert.strictEqual(typeof type, 'string'); assert.strictEqual(typeof type, 'string');
assert(Array.isArray(values)); assert(Array.isArray(values));
debug('upsert: %s for zone %s of type %s with values %j', location, domainObject.zoneName, type, values); log('upsert: %s for zone %s of type %s with values %j', location, domainObject.zoneName, type, values);
return; return;
} }

20
src/dns/ovh.js
View File

@@ -1,14 +1,14 @@
import assert from 'node:assert'; import assert from 'node:assert';
import BoxError from '../boxerror.js'; import BoxError from '../boxerror.js';
import constants from '../constants.js'; import constants from '../constants.js';
import debugModule from 'debug'; import logger from '../logger.js';
import dig from '../dig.js'; import dig from '../dig.js';
import dns from '../dns.js'; import dns from '../dns.js';
import ovhClient from 'ovh'; import ovhClient from 'ovh';
import safe from 'safetydance'; import safe from 'safetydance';
import waitForDns from './waitfordns.js'; import waitForDns from './waitfordns.js';
const debug = debugModule('box:dns/ovh'); const { log, trace } = logger('dns/ovh');
function formatError(error) { function formatError(error) {
@@ -39,7 +39,7 @@ async function getDnsRecordIds(domainConfig, zoneName, name, type) {
assert.strictEqual(typeof name, 'string'); assert.strictEqual(typeof name, 'string');
assert.strictEqual(typeof type, 'string'); assert.strictEqual(typeof type, 'string');
debug(`get: ${name} in zone ${zoneName} of type ${type}`); log(`get: ${name} in zone ${zoneName} of type ${type}`);
const client = createClient(domainConfig); const client = createClient(domainConfig);
const [error, data] = await safe(client.requestPromised('GET', `/domain/zone/${zoneName}/record`, { fieldType: type, subDomain: name })); const [error, data] = await safe(client.requestPromised('GET', `/domain/zone/${zoneName}/record`, { fieldType: type, subDomain: name }));
@@ -54,7 +54,7 @@ async function refreshZone(domainConfig, zoneName) {
assert.strictEqual(typeof domainConfig, 'object'); assert.strictEqual(typeof domainConfig, 'object');
assert.strictEqual(typeof zoneName, 'string'); assert.strictEqual(typeof zoneName, 'string');
debug(`refresh: zone ${zoneName}`); log(`refresh: zone ${zoneName}`);
const client = createClient(domainConfig); const client = createClient(domainConfig);
const [error] = await safe(client.requestPromised('POST', `/domain/zone/${zoneName}/refresh`)); const [error] = await safe(client.requestPromised('POST', `/domain/zone/${zoneName}/refresh`));
@@ -74,7 +74,7 @@ async function upsert(domainObject, location, type, values) {
zoneName = domainObject.zoneName, zoneName = domainObject.zoneName,
name = dns.getName(domainObject, location, type) || ''; name = dns.getName(domainObject, location, type) || '';
debug(`upsert: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`); log(`upsert: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
const recordIds = await getDnsRecordIds(domainConfig, zoneName, name, type); const recordIds = await getDnsRecordIds(domainConfig, zoneName, name, type);
@@ -115,7 +115,7 @@ async function upsert(domainObject, location, type, values) {
} }
await refreshZone(domainConfig, zoneName); await refreshZone(domainConfig, zoneName);
debug('upsert: completed'); log('upsert: completed');
} }
async function get(domainObject, location, type) { async function get(domainObject, location, type) {
@@ -152,7 +152,7 @@ async function del(domainObject, location, type, values) {
zoneName = domainObject.zoneName, zoneName = domainObject.zoneName,
name = dns.getName(domainObject, location, type) || ''; name = dns.getName(domainObject, location, type) || '';
debug(`del: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`); log(`del: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
const recordIds = await getDnsRecordIds(domainConfig, zoneName, name, type); const recordIds = await getDnsRecordIds(domainConfig, zoneName, name, type);
@@ -211,17 +211,17 @@ async function verifyDomainConfig(domainObject) {
// ovh.net, ovh.ca or anycast.me // ovh.net, ovh.ca or anycast.me
if (!nameservers.every(function (n) { return n.toLowerCase().search(/ovh|kimsufi|anycast/) !== -1; })) { if (!nameservers.every(function (n) { return n.toLowerCase().search(/ovh|kimsufi|anycast/) !== -1; })) {
debug('verifyDomainConfig: %j does not contain OVH NS', nameservers); log('verifyDomainConfig: %j does not contain OVH NS', nameservers);
if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to OVH'); if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to OVH');
} }
const location = 'cloudrontestdns'; const location = 'cloudrontestdns';
await upsert(domainObject, location, 'A', [ ip ]); await upsert(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record added'); log('verifyDomainConfig: Test A record added');
await del(domainObject, location, 'A', [ ip ]); await del(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record removed again'); log('verifyDomainConfig: Test A record removed again');
return credentials; return credentials;
} }

18
src/dns/porkbun.js
View File

@@ -1,7 +1,7 @@
import assert from 'node:assert'; import assert from 'node:assert';
import BoxError from '../boxerror.js'; import BoxError from '../boxerror.js';
import constants from '../constants.js'; import constants from '../constants.js';
import debugModule from 'debug'; import logger from '../logger.js';
import dig from '../dig.js'; import dig from '../dig.js';
import dns from '../dns.js'; import dns from '../dns.js';
import safe from 'safetydance'; import safe from 'safetydance';
@@ -9,7 +9,7 @@ import superagent from '@cloudron/superagent';
import timers from 'timers/promises'; import timers from 'timers/promises';
import waitForDns from './waitfordns.js'; import waitForDns from './waitfordns.js';
const debug = debugModule('box:dns/porkbun'); const { log, trace } = logger('dns/porkbun');
// Rate limit note: Porkbun return 503 when it hits rate limits. It's as low as 1 req/second // Rate limit note: Porkbun return 503 when it hits rate limits. It's as low as 1 req/second
@@ -44,7 +44,7 @@ async function getDnsRecords(domainConfig, zoneName, name, type) {
assert.strictEqual(typeof name, 'string'); assert.strictEqual(typeof name, 'string');
assert.strictEqual(typeof type, 'string'); assert.strictEqual(typeof type, 'string');
debug(`get: ${name} zone:${zoneName} type:${type}`); log(`get: ${name} zone:${zoneName} type:${type}`);
const data = { const data = {
secretapikey: domainConfig.secretapikey, secretapikey: domainConfig.secretapikey,
@@ -90,7 +90,7 @@ async function upsert(domainObject, location, type, values) {
zoneName = domainObject.zoneName, zoneName = domainObject.zoneName,
name = dns.getName(domainObject, location, type) || ''; name = dns.getName(domainObject, location, type) || '';
debug(`upsert: ${name} zone:${zoneName} type:${type} values:${JSON.stringify(values)}`); log(`upsert: ${name} zone:${zoneName} type:${type} values:${JSON.stringify(values)}`);
await delDnsRecords(domainConfig, zoneName, name, type); await delDnsRecords(domainConfig, zoneName, name, type);
@@ -119,7 +119,7 @@ async function upsert(domainObject, location, type, values) {
if (response.body.status !== 'SUCCESS') throw new BoxError(BoxError.EXTERNAL_ERROR, `Invalid status in response: ${JSON.stringify(response.body)}`); if (response.body.status !== 'SUCCESS') throw new BoxError(BoxError.EXTERNAL_ERROR, `Invalid status in response: ${JSON.stringify(response.body)}`);
if (!response.body.id) throw new BoxError(BoxError.EXTERNAL_ERROR, `Invalid id in response: ${JSON.stringify(response.body)}`); if (!response.body.id) throw new BoxError(BoxError.EXTERNAL_ERROR, `Invalid id in response: ${JSON.stringify(response.body)}`);
debug(`upsert: created record with id ${response.body.id}`); log(`upsert: created record with id ${response.body.id}`);
} }
} }
@@ -146,7 +146,7 @@ async function del(domainObject, location, type, values) {
zoneName = domainObject.zoneName, zoneName = domainObject.zoneName,
name = dns.getName(domainObject, location, type) || ''; name = dns.getName(domainObject, location, type) || '';
debug(`del: ${name} zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`); log(`del: ${name} zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
const data = { const data = {
secretapikey: domainConfig.secretapikey, secretapikey: domainConfig.secretapikey,
@@ -203,17 +203,17 @@ async function verifyDomainConfig(domainObject) {
if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers'); if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers');
if (!nameservers.every(function (n) { return n.toLowerCase().indexOf('.ns.porkbun.com') !== -1; })) { if (!nameservers.every(function (n) { return n.toLowerCase().indexOf('.ns.porkbun.com') !== -1; })) {
debug('verifyDomainConfig: %j does not contain Porkbun NS', nameservers); log('verifyDomainConfig: %j does not contain Porkbun NS', nameservers);
if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Porkbun'); if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Porkbun');
} }
const location = 'cloudrontestdns'; const location = 'cloudrontestdns';
await upsert(domainObject, location, 'A', [ ip ]); await upsert(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record added'); log('verifyDomainConfig: Test A record added');
await del(domainObject, location, 'A', [ ip ]); await del(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record removed again'); log('verifyDomainConfig: Test A record removed again');
return credentials; return credentials;
} }

14
src/dns/route53.js
View File

@@ -2,7 +2,7 @@ import assert from 'node:assert';
import BoxError from '../boxerror.js'; import BoxError from '../boxerror.js';
import { ConfiguredRetryStrategy } from '@smithy/util-retry'; import { ConfiguredRetryStrategy } from '@smithy/util-retry';
import constants from '../constants.js'; import constants from '../constants.js';
import debugModule from 'debug'; import logger from '../logger.js';
import dig from '../dig.js'; import dig from '../dig.js';
import dns from '../dns.js'; import dns from '../dns.js';
import { Route53 } from '@aws-sdk/client-route-53'; import { Route53 } from '@aws-sdk/client-route-53';
@@ -10,7 +10,7 @@ import safe from 'safetydance';
import waitForDns from './waitfordns.js'; import waitForDns from './waitfordns.js';
import _ from '../underscore.js'; import _ from '../underscore.js';
const debug = debugModule('box:dns/route53'); const { log, trace } = logger('dns/route53');
function removePrivateFields(domainObject) { function removePrivateFields(domainObject) {
@@ -93,7 +93,7 @@ async function upsert(domainObject, location, type, values) {
zoneName = domainObject.zoneName, zoneName = domainObject.zoneName,
fqdn = dns.fqdn(location, domainObject.domain); fqdn = dns.fqdn(location, domainObject.domain);
debug('add: %s for zone %s of type %s with values %j', fqdn, zoneName, type, values); log('add: %s for zone %s of type %s with values %j', fqdn, zoneName, type, values);
const zone = await getZoneByName(domainConfig, zoneName); const zone = await getZoneByName(domainConfig, zoneName);
@@ -243,7 +243,7 @@ async function verifyDomainConfig(domainObject) {
const zone = await getHostedZone(credentials, zoneName); const zone = await getHostedZone(credentials, zoneName);
if (!_.isEqual(zone.DelegationSet.NameServers.sort(), nameservers.sort())) { if (!_.isEqual(zone.DelegationSet.NameServers.sort(), nameservers.sort())) {
debug('verifyDomainConfig: %j and %j do not match', nameservers, zone.DelegationSet.NameServers); log('verifyDomainConfig: %j and %j do not match', nameservers, zone.DelegationSet.NameServers);
if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Route53'); if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Route53');
} }
@@ -251,13 +251,13 @@ async function verifyDomainConfig(domainObject) {
const newDomainObject = Object.assign({ }, domainObject, { config: credentials }); const newDomainObject = Object.assign({ }, domainObject, { config: credentials });
await upsert(newDomainObject, location, 'A', [ ip ]); await upsert(newDomainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record added'); log('verifyDomainConfig: Test A record added');
await get(newDomainObject, location, 'A'); await get(newDomainObject, location, 'A');
debug('verifyDomainConfig: Can list record sets'); log('verifyDomainConfig: Can list record sets');
await del(newDomainObject, location, 'A', [ ip ]); await del(newDomainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record removed again'); log('verifyDomainConfig: Test A record removed again');
return credentials; return credentials;
} }

18
src/dns/vultr.js
View File

@@ -1,14 +1,14 @@
import assert from 'node:assert'; import assert from 'node:assert';
import constants from '../constants.js'; import constants from '../constants.js';
import BoxError from '../boxerror.js'; import BoxError from '../boxerror.js';
import debugModule from 'debug'; import logger from '../logger.js';
import dig from '../dig.js'; import dig from '../dig.js';
import dns from '../dns.js'; import dns from '../dns.js';
import safe from 'safetydance'; import safe from 'safetydance';
import superagent from '@cloudron/superagent'; import superagent from '@cloudron/superagent';
import waitForDns from './waitfordns.js'; import waitForDns from './waitfordns.js';
const debug = debugModule('box:dns/vultr'); const { log, trace } = logger('dns/vultr');
const VULTR_ENDPOINT = 'https://api.vultr.com/v2'; const VULTR_ENDPOINT = 'https://api.vultr.com/v2';
@@ -32,7 +32,7 @@ async function getZoneRecords(domainConfig, zoneName, name, type) {
assert.strictEqual(typeof name, 'string'); assert.strictEqual(typeof name, 'string');
assert.strictEqual(typeof type, 'string'); assert.strictEqual(typeof type, 'string');
debug(`getInternal: getting dns records of ${zoneName} with ${name} and type ${type}`); log(`getInternal: getting dns records of ${zoneName} with ${name} and type ${type}`);
const per_page = 100; const per_page = 100;
let cursor = null, records = []; let cursor = null, records = [];
@@ -80,7 +80,7 @@ async function upsert(domainObject, location, type, values) {
zoneName = domainObject.zoneName, zoneName = domainObject.zoneName,
name = dns.getName(domainObject, location, type) || ''; name = dns.getName(domainObject, location, type) || '';
debug('upsert: %s for zone %s of type %s with values %j', name, zoneName, type, values); log('upsert: %s for zone %s of type %s with values %j', name, zoneName, type, values);
const records = await getZoneRecords(domainConfig, zoneName, name, type); const records = await getZoneRecords(domainConfig, zoneName, name, type);
@@ -143,10 +143,10 @@ async function upsert(domainObject, location, type, values) {
.timeout(30 * 1000) .timeout(30 * 1000)
.retry(5)); .retry(5));
if (error) debug(`upsert: error removing record ${records[j].id}: ${error.message}`); if (error) log(`upsert: error removing record ${records[j].id}: ${error.message}`);
} }
debug('upsert: completed with recordIds:%j', recordIds); log('upsert: completed with recordIds:%j', recordIds);
} }
async function del(domainObject, location, type, values) { async function del(domainObject, location, type, values) {
@@ -214,17 +214,17 @@ async function verifyDomainConfig(domainObject) {
if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers'); if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers');
if (nameservers.map(function (n) { return n.toLowerCase(); }).indexOf('ns1.vultr.com') === -1) { if (nameservers.map(function (n) { return n.toLowerCase(); }).indexOf('ns1.vultr.com') === -1) {
debug('verifyDomainConfig: %j does not contains vultr NS', nameservers); log('verifyDomainConfig: %j does not contains vultr NS', nameservers);
if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Vultr'); if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Vultr');
} }
const location = 'cloudrontestdns'; const location = 'cloudrontestdns';
await upsert(domainObject, location, 'A', [ ip ]); await upsert(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record added'); log('verifyDomainConfig: Test A record added');
await del(domainObject, location, 'A', [ ip ]); await del(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record removed again'); log('verifyDomainConfig: Test A record removed again');
return credentials; return credentials;
} }

28
src/dns/waitfordns.js
View File

@@ -1,13 +1,13 @@
import assert from 'node:assert'; import assert from 'node:assert';
import BoxError from '../boxerror.js'; import BoxError from '../boxerror.js';
import debugModule from 'debug'; import logger from '../logger.js';
import dig from '../dig.js'; import dig from '../dig.js';
import dns from 'node:dns'; import dns from 'node:dns';
import promiseRetry from '../promise-retry.js'; import promiseRetry from '../promise-retry.js';
import safe from 'safetydance'; import safe from 'safetydance';
import _ from '../underscore.js'; import _ from '../underscore.js';
const debug = debugModule('box:dns/waitfordns'); const { log, trace } = logger('dns/waitfordns');
async function resolveIp(hostname, type, options) { async function resolveIp(hostname, type, options) {
assert.strictEqual(typeof hostname, 'string'); assert.strictEqual(typeof hostname, 'string');
@@ -15,17 +15,17 @@ async function resolveIp(hostname, type, options) {
assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof options, 'object');
// try A record at authoritative server // try A record at authoritative server
debug(`resolveIp: Checking ${type} for ${hostname} at ${options.server}`); log(`resolveIp: Checking ${type} for ${hostname} at ${options.server}`);
const [error, results] = await safe(dig.resolve(hostname, type, options)); const [error, results] = await safe(dig.resolve(hostname, type, options));
if (!error && results.length !== 0) return results; if (!error && results.length !== 0) return results;
// try CNAME record at authoritative server // try CNAME record at authoritative server
debug(`resolveIp: No ${type}. Checking CNAME for ${hostname} at ${options.server}`); log(`resolveIp: No ${type}. Checking CNAME for ${hostname} at ${options.server}`);
const cnameResults = await dig.resolve(hostname, 'CNAME', options); const cnameResults = await dig.resolve(hostname, 'CNAME', options);
if (cnameResults.length === 0) return cnameResults; if (cnameResults.length === 0) return cnameResults;
// recurse lookup the CNAME record // recurse lookup the CNAME record
debug(`resolveIp: found CNAME for ${hostname}. resolving ${cnameResults[0]}`); log(`resolveIp: found CNAME for ${hostname}. resolving ${cnameResults[0]}`);
return await dig.resolve(cnameResults[0], type, _.omit(options, ['server'])); return await dig.resolve(cnameResults[0], type, _.omit(options, ['server']));
} }
@@ -40,7 +40,7 @@ async function isChangeSynced(hostname, type, value, nameserver) {
const [error6, nsIPv6s] = await safe(dig.resolve(nameserver, 'AAAA', { timeout: 5000 })); const [error6, nsIPv6s] = await safe(dig.resolve(nameserver, 'AAAA', { timeout: 5000 }));
if (error4 && error6) { if (error4 && error6) {
debug(`isChangeSynced: cannot resolve NS ${nameserver}`); // NS doesn't resolve at all; it's fine log(`isChangeSynced: cannot resolve NS ${nameserver}`); // NS doesn't resolve at all; it's fine
return true; return true;
} }
@@ -54,13 +54,13 @@ async function isChangeSynced(hostname, type, value, nameserver) {
const [error, answer] = await safe(resolver); const [error, answer] = await safe(resolver);
// CONNREFUSED - when there is no ipv4/ipv6 connectivity. REFUSED - server won't answer maybe by policy // CONNREFUSED - when there is no ipv4/ipv6 connectivity. REFUSED - server won't answer maybe by policy
if (error && (error.code === dns.TIMEOUT || error.code === dns.REFUSED || error.code === dns.CONNREFUSED)) { if (error && (error.code === dns.TIMEOUT || error.code === dns.REFUSED || error.code === dns.CONNREFUSED)) {
debug(`isChangeSynced: NS ${nameserver} (${nsIp}) not resolving ${hostname} (${type}): ${error}. Ignoring`); log(`isChangeSynced: NS ${nameserver} (${nsIp}) not resolving ${hostname} (${type}): ${error}. Ignoring`);
status[i] = true; // should be ok if dns server is down status[i] = true; // should be ok if dns server is down
continue; continue;
} }
if (error) { if (error) {
debug(`isChangeSynced: NS ${nameserver} (${nsIp}) errored when resolve ${hostname} (${type}): ${error}`); log(`isChangeSynced: NS ${nameserver} (${nsIp}) errored when resolve ${hostname} (${type}): ${error}`);
status[i] = false; status[i] = false;
continue; continue;
} }
@@ -72,7 +72,7 @@ async function isChangeSynced(hostname, type, value, nameserver) {
match = answer.some(function (a) { return value === a.join(''); }); match = answer.some(function (a) { return value === a.join(''); });
} }
debug(`isChangeSynced: ${hostname} (${type}) was resolved to ${answer} at NS ${nameserver} (${nsIp}). Expecting ${value}. Match ${match}`); log(`isChangeSynced: ${hostname} (${type}) was resolved to ${answer} at NS ${nameserver} (${nsIp}). Expecting ${value}. Match ${match}`);
status[i] = match; status[i] = match;
} }
@@ -87,21 +87,21 @@ async function waitForDns(hostname, zoneName, type, value, options) {
assert.strictEqual(typeof value, 'string'); assert.strictEqual(typeof value, 'string');
assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 } assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
debug(`waitForDns: waiting for ${hostname} to be ${value} in zone ${zoneName}`); log(`waitForDns: waiting for ${hostname} to be ${value} in zone ${zoneName}`);
await promiseRetry(Object.assign({ debug }, options), async function () { await promiseRetry(Object.assign({ debug: log }, options), async function () {
const nameservers = await dig.resolve(zoneName, 'NS', { timeout: 5000 }); const nameservers = await dig.resolve(zoneName, 'NS', { timeout: 5000 });
if (!nameservers) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Unable to get nameservers'); if (!nameservers) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Unable to get nameservers');
debug(`waitForDns: nameservers are ${JSON.stringify(nameservers)}`); log(`waitForDns: nameservers are ${JSON.stringify(nameservers)}`);
for (const nameserver of nameservers) { for (const nameserver of nameservers) {
const synced = await isChangeSynced(hostname, type, value, nameserver); const synced = await isChangeSynced(hostname, type, value, nameserver);
debug(`waitForDns: ${hostname} at ns ${nameserver}: ${synced ? 'done' : 'not done'} `); log(`waitForDns: ${hostname} at ns ${nameserver}: ${synced ? 'done' : 'not done'} `);
if (!synced) throw new BoxError(BoxError.EXTERNAL_ERROR, 'ETRYAGAIN'); if (!synced) throw new BoxError(BoxError.EXTERNAL_ERROR, 'ETRYAGAIN');
} }
}); });
debug(`waitForDns: ${hostname} has propagated`); log(`waitForDns: ${hostname} has propagated`);
} }
export default waitForDns; export default waitForDns;

6
src/dns/wildcard.js
View File

@@ -1,13 +1,13 @@
import assert from 'node:assert'; import assert from 'node:assert';
import BoxError from '../boxerror.js'; import BoxError from '../boxerror.js';
import debugModule from 'debug'; import logger from '../logger.js';
import dig from '../dig.js'; import dig from '../dig.js';
import dns from '../dns.js'; import dns from '../dns.js';
import network from '../network.js'; import network from '../network.js';
import safe from 'safetydance'; import safe from 'safetydance';
import waitForDns from './waitfordns.js'; import waitForDns from './waitfordns.js';
const debug = debugModule('box:dns/manual'); const { log, trace } = logger('dns/manual');
function removePrivateFields(domainObject) { function removePrivateFields(domainObject) {
@@ -24,7 +24,7 @@ async function upsert(domainObject, location, type, values) {
assert.strictEqual(typeof type, 'string'); assert.strictEqual(typeof type, 'string');
assert(Array.isArray(values)); assert(Array.isArray(values));
debug('upsert: %s for zone %s of type %s with values %j', location, domainObject.zoneName, type, values); log('upsert: %s for zone %s of type %s with values %j', location, domainObject.zoneName, type, values);
return; return;
} }

46
src/docker.js
View File

@@ -3,7 +3,7 @@ import assert from 'node:assert';
import BoxError from './boxerror.js'; import BoxError from './boxerror.js';
import constants from './constants.js'; import constants from './constants.js';
import dashboard from './dashboard.js'; import dashboard from './dashboard.js';
import debugModule from 'debug'; import logger from './logger.js';
import Docker from 'dockerode'; import Docker from 'dockerode';
import dockerRegistries from './dockerregistries.js'; import dockerRegistries from './dockerregistries.js';
import fs from 'node:fs'; import fs from 'node:fs';
@@ -17,7 +17,7 @@ import safe from 'safetydance';
import timers from 'timers/promises'; import timers from 'timers/promises';
import volumes from './volumes.js'; import volumes from './volumes.js';
const debug = debugModule('box:docker'); const { log, trace } = logger('docker');
const shell = shellModule('docker'); const shell = shellModule('docker');
@@ -94,7 +94,7 @@ async function pullImage(imageRef) {
const authConfig = await getAuthConfig(imageRef); const authConfig = await getAuthConfig(imageRef);
debug(`pullImage: will pull ${imageRef}. auth: ${authConfig ? 'yes' : 'no'}`); log(`pullImage: will pull ${imageRef}. auth: ${authConfig ? 'yes' : 'no'}`);
const [error, stream] = await safe(gConnection.pull(imageRef, { authconfig: authConfig })); const [error, stream] = await safe(gConnection.pull(imageRef, { authconfig: authConfig }));
if (error && error.statusCode === 404) throw new BoxError(BoxError.NOT_FOUND, `Unable to pull image ${imageRef}. message: ${error.message} statusCode: ${error.statusCode}`); if (error && error.statusCode === 404) throw new BoxError(BoxError.NOT_FOUND, `Unable to pull image ${imageRef}. message: ${error.message} statusCode: ${error.statusCode}`);
@@ -107,17 +107,17 @@ async function pullImage(imageRef) {
let layerError = null; let layerError = null;
stream.on('data', function (chunk) { stream.on('data', function (chunk) {
const data = safe.JSON.parse(chunk) || { }; const data = safe.JSON.parse(chunk) || { };
debug('pullImage: %j', data); log('pullImage: %j', data);
// The data.status here is useless because this is per layer as opposed to per image // The data.status here is useless because this is per layer as opposed to per image
if (!data.status && data.error) { // data is { errorDetail: { message: xx } , error: xx } if (!data.status && data.error) { // data is { errorDetail: { message: xx } , error: xx }
debug(`pullImage error ${imageRef}: ${data.errorDetail.message}`); log(`pullImage error ${imageRef}: ${data.errorDetail.message}`);
layerError = data.errorDetail; layerError = data.errorDetail;
} }
}); });
stream.on('end', function () { stream.on('end', function () {
debug(`downloaded image ${imageRef} . error: ${!!layerError}`); log(`downloaded image ${imageRef} . error: ${!!layerError}`);
if (!layerError) return resolve(); if (!layerError) return resolve();
@@ -125,7 +125,7 @@ async function pullImage(imageRef) {
}); });
stream.on('error', function (streamError) { // this is only hit for stream error and not for some download error stream.on('error', function (streamError) { // this is only hit for stream error and not for some download error
debug(`error pulling image ${imageRef}: %o`, streamError); log(`error pulling image ${imageRef}: %o`, streamError);
reject(new BoxError(BoxError.DOCKER_ERROR, streamError.message)); reject(new BoxError(BoxError.DOCKER_ERROR, streamError.message));
}); });
}); });
@@ -135,7 +135,7 @@ async function buildImage(dockerImage, sourceArchiveFilePath) {
assert.strictEqual(typeof dockerImage, 'string'); assert.strictEqual(typeof dockerImage, 'string');
assert.strictEqual(typeof sourceArchiveFilePath, 'string'); assert.strictEqual(typeof sourceArchiveFilePath, 'string');
debug(`buildImage: building ${dockerImage} from ${sourceArchiveFilePath}`); log(`buildImage: building ${dockerImage} from ${sourceArchiveFilePath}`);
const buildOptions = { t: dockerImage }; const buildOptions = { t: dockerImage };
const [listError, listOut] = await safe(shell.spawn('tar', ['-tzf', sourceArchiveFilePath], { encoding: 'utf8' })); const [listError, listOut] = await safe(shell.spawn('tar', ['-tzf', sourceArchiveFilePath], { encoding: 'utf8' }));
@@ -146,7 +146,7 @@ async function buildImage(dockerImage, sourceArchiveFilePath) {
}); });
if (dockerfileCloudronPath) { if (dockerfileCloudronPath) {
buildOptions.dockerfile = dockerfileCloudronPath.replace(/\/$/, ''); buildOptions.dockerfile = dockerfileCloudronPath.replace(/\/$/, '');
debug(`buildImage: using ${buildOptions.dockerfile}`); log(`buildImage: using ${buildOptions.dockerfile}`);
} }
} }
@@ -164,23 +164,23 @@ async function buildImage(dockerImage, sourceArchiveFilePath) {
buildError = data.errorDetail || { message: data.error }; buildError = data.errorDetail || { message: data.error };
} else { } else {
const message = (data.stream || data.status || data.aux?.ID || '').replace(/\n$/, ''); const message = (data.stream || data.status || data.aux?.ID || '').replace(/\n$/, '');
if (message) debug('buildImage: ' + message); if (message) log('buildImage: ' + message);
} }
}); });
stream.on('end', () => { stream.on('end', () => {
if (buildError) { if (buildError) {
debug(`buildImage: error ${buildError}`); log(`buildImage: error ${buildError}`);
return reject(new BoxError(buildError.message.includes('no space') ? BoxError.FS_ERROR : BoxError.DOCKER_ERROR, buildError.message)); return reject(new BoxError(buildError.message.includes('no space') ? BoxError.FS_ERROR : BoxError.DOCKER_ERROR, buildError.message));
} else { } else {
debug(`buildImage: success ${dockerImage}`); log(`buildImage: success ${dockerImage}`);
} }
resolve(); resolve();
}); });
stream.on('error', (streamError) => { stream.on('error', (streamError) => {
debug(`buildImage: error building image ${dockerImage}: %o`, streamError); log(`buildImage: error building image ${dockerImage}: %o`, streamError);
reject(new BoxError(BoxError.DOCKER_ERROR, streamError.message)); reject(new BoxError(BoxError.DOCKER_ERROR, streamError.message));
}); });
}); });
@@ -329,7 +329,7 @@ async function restartContainer(containerId) {
async function stopContainer(containerId) { async function stopContainer(containerId) {
assert.strictEqual(typeof containerId, 'string'); assert.strictEqual(typeof containerId, 'string');
debug(`stopContainer: stopping container ${containerId}`); log(`stopContainer: stopping container ${containerId}`);
const container = gConnection.getContainer(containerId); const container = gConnection.getContainer(containerId);
@@ -347,7 +347,7 @@ async function stopContainer(containerId) {
async function deleteContainer(containerId) { // id can also be name async function deleteContainer(containerId) { // id can also be name
assert.strictEqual(typeof containerId, 'string'); assert.strictEqual(typeof containerId, 'string');
debug(`deleteContainer: deleting ${containerId}`); log(`deleteContainer: deleting ${containerId}`);
const container = gConnection.getContainer(containerId); const container = gConnection.getContainer(containerId);
@@ -360,7 +360,7 @@ async function deleteContainer(containerId) { // id can also be name
if (error && error.statusCode === 404) return; if (error && error.statusCode === 404) return;
if (error) { if (error) {
debug('Error removing container %s : %o', containerId, error); log('Error removing container %s : %o', containerId, error);
throw new BoxError(BoxError.DOCKER_ERROR, error); throw new BoxError(BoxError.DOCKER_ERROR, error);
} }
} }
@@ -405,14 +405,14 @@ async function deleteImage(imageRef) {
// registry v1 used to pull down all *tags*. this meant that deleting image by tag was not enough (since that // registry v1 used to pull down all *tags*. this meant that deleting image by tag was not enough (since that
// just removes the tag). we used to remove the image by id. this is not required anymore because aliases are // just removes the tag). we used to remove the image by id. this is not required anymore because aliases are
// not created anymore after https://github.com/docker/docker/pull/10571 // not created anymore after https://github.com/docker/docker/pull/10571
debug(`deleteImage: removing ${imageRef}`); log(`deleteImage: removing ${imageRef}`);
const [error] = await safe(gConnection.getImage(imageRef.replace(/@sha256:.*/,'')).remove(removeOptions)); // can't have the manifest id. won't remove anythin const [error] = await safe(gConnection.getImage(imageRef.replace(/@sha256:.*/,'')).remove(removeOptions)); // can't have the manifest id. won't remove anythin
if (error && error.statusCode === 400) return; // invalid image format. this can happen if user installed with a bad --docker-image if (error && error.statusCode === 400) return; // invalid image format. this can happen if user installed with a bad --docker-image
if (error && error.statusCode === 404) return; // not found if (error && error.statusCode === 404) return; // not found
if (error && error.statusCode === 409) return; // another container using the image if (error && error.statusCode === 409) return; // another container using the image
if (error) { if (error) {
debug(`Error removing image ${imageRef} : %o`, error); log(`Error removing image ${imageRef} : %o`, error);
throw new BoxError(BoxError.DOCKER_ERROR, error); throw new BoxError(BoxError.DOCKER_ERROR, error);
} }
} }
@@ -432,7 +432,7 @@ async function inspect(containerId) {
async function downloadImage(manifest) { async function downloadImage(manifest) {
assert.strictEqual(typeof manifest, 'object'); assert.strictEqual(typeof manifest, 'object');
debug(`downloadImage: ${manifest.dockerImage}`); log(`downloadImage: ${manifest.dockerImage}`);
const image = gConnection.getImage(manifest.dockerImage); const image = gConnection.getImage(manifest.dockerImage);
@@ -441,7 +441,7 @@ async function downloadImage(manifest) {
const parsedManifestRef = parseImageRef(manifest.dockerImage); const parsedManifestRef = parseImageRef(manifest.dockerImage);
await promiseRetry({ times: 10, interval: 5000, debug, retry: (pullError) => pullError.reason !== BoxError.FS_ERROR }, async () => { await promiseRetry({ times: 10, interval: 5000, debug: log, retry: (pullError) => pullError.reason !== BoxError.FS_ERROR }, async () => {
// custom (non appstore) image // custom (non appstore) image
if (parsedManifestRef.registry !== null || !parsedManifestRef.fullRepositoryName.startsWith('cloudron/')) return await pullImage(manifest.dockerImage); if (parsedManifestRef.registry !== null || !parsedManifestRef.fullRepositoryName.startsWith('cloudron/')) return await pullImage(manifest.dockerImage);
@@ -457,9 +457,9 @@ async function downloadImage(manifest) {
if (pullError || !upstreamRef) throw new BoxError(BoxError.DOCKER_ERROR, `Unable to pull ${manifest.dockerImage} from dockerhub or quay: ${pullError?.message}`); if (pullError || !upstreamRef) throw new BoxError(BoxError.DOCKER_ERROR, `Unable to pull ${manifest.dockerImage} from dockerhub or quay: ${pullError?.message}`);
// retag the downloaded image to not have the registry name. this prevents 'docker run' from redownloading it // retag the downloaded image to not have the registry name. this prevents 'docker run' from redownloading it
debug(`downloadImage: tagging ${upstreamRef} as ${parsedManifestRef.fullRepositoryName}:${parsedManifestRef.tag}`); log(`downloadImage: tagging ${upstreamRef} as ${parsedManifestRef.fullRepositoryName}:${parsedManifestRef.tag}`);
await gConnection.getImage(upstreamRef).tag({ repo: parsedManifestRef.fullRepositoryName, tag: parsedManifestRef.tag }); await gConnection.getImage(upstreamRef).tag({ repo: parsedManifestRef.fullRepositoryName, tag: parsedManifestRef.tag });
debug(`downloadImage: untagging ${upstreamRef}`); log(`downloadImage: untagging ${upstreamRef}`);
await deleteImage(upstreamRef); await deleteImage(upstreamRef);
}); });
} }
@@ -731,7 +731,7 @@ async function createSubcontainer(app, name, cmd, options) {
containerOptions.HostConfig.Devices = Object.keys(app.devices).map((d) => { containerOptions.HostConfig.Devices = Object.keys(app.devices).map((d) => {
if (!safe.fs.existsSync(d)) { if (!safe.fs.existsSync(d)) {
debug(`createSubcontainer: device ${d} does not exist. Skipping...`); log(`createSubcontainer: device ${d} does not exist. Skipping...`);
return null; return null;
} }

16
src/dockerproxy.js
View File

@@ -2,7 +2,7 @@ import apps from './apps.js';
import assert from 'node:assert'; import assert from 'node:assert';
import constants from './constants.js'; import constants from './constants.js';
import express from 'express'; import express from 'express';
import debugModule from 'debug'; import logger from './logger.js';
import http from 'node:http'; import http from 'node:http';
import { HttpError } from '@cloudron/connect-lastmile'; import { HttpError } from '@cloudron/connect-lastmile';
import middleware from './middleware/index.js'; import middleware from './middleware/index.js';
@@ -13,7 +13,7 @@ import safe from 'safetydance';
import util from 'node:util'; import util from 'node:util';
import volumes from './volumes.js'; import volumes from './volumes.js';
const debug = debugModule('box:dockerproxy'); const { log, trace } = logger('dockerproxy');
let gHttpServer = null; let gHttpServer = null;
@@ -49,7 +49,7 @@ function attachDockerRequest(req, res, next) {
// Force node to send out the headers, this is required for the /container/wait api to make the docker cli proceed // Force node to send out the headers, this is required for the /container/wait api to make the docker cli proceed
res.write(' '); res.write(' ');
dockerResponse.on('error', function (error) { debug('dockerResponse error: %o', error); }); dockerResponse.on('error', function (error) { log('dockerResponse error: %o', error); });
dockerResponse.pipe(res, { end: true }); dockerResponse.pipe(res, { end: true });
}); });
@@ -66,7 +66,7 @@ async function containersCreate(req, res, next) {
const appDataDir = path.join(paths.APPS_DATA_DIR, req.resources.app.id, 'data'); const appDataDir = path.join(paths.APPS_DATA_DIR, req.resources.app.id, 'data');
debug('containersCreate: original bind mounts:', req.body.HostConfig.Binds); log('containersCreate: original bind mounts:', req.body.HostConfig.Binds);
const [error, result] = await safe(volumes.list()); const [error, result] = await safe(volumes.list());
if (error) return next(new HttpError(500, `Error listing volumes: ${error.message}`)); if (error) return next(new HttpError(500, `Error listing volumes: ${error.message}`));
@@ -82,14 +82,14 @@ async function containersCreate(req, res, next) {
const volumeName = bind.match(new RegExp('/media/([^:/]+)/?'))[1]; const volumeName = bind.match(new RegExp('/media/([^:/]+)/?'))[1];
const volume = volumesByName[volumeName]; const volume = volumesByName[volumeName];
if (volume) binds.push(bind.replace(new RegExp(`^/media/${volumeName}`), volume.hostPath)); if (volume) binds.push(bind.replace(new RegExp(`^/media/${volumeName}`), volume.hostPath));
else debug(`containersCreate: dropped unknown volume ${volumeName}`); else log(`containersCreate: dropped unknown volume ${volumeName}`);
} else { } else {
req.dockerRequest.abort(); req.dockerRequest.abort();
return next(new HttpError(400, 'Binds must be under /app/data/ or /media')); return next(new HttpError(400, 'Binds must be under /app/data/ or /media'));
} }
} }
debug('containersCreate: rewritten bind mounts:', binds); log('containersCreate: rewritten bind mounts:', binds);
safe.set(req.body, 'HostConfig.Binds', binds); safe.set(req.body, 'HostConfig.Binds', binds);
const plainBody = JSON.stringify(req.body); const plainBody = JSON.stringify(req.body);
@@ -125,7 +125,7 @@ async function start() {
if (constants.TEST) { if (constants.TEST) {
proxyServer.use(function (req, res, next) { proxyServer.use(function (req, res, next) {
debug('proxying: ' + req.method, req.url); log('proxying: ' + req.method, req.url);
next(); next();
}); });
} }
@@ -177,7 +177,7 @@ async function start() {
}); });
}); });
debug(`start: listening on 172.18.0.1:${constants.DOCKER_PROXY_PORT}`); log(`start: listening on 172.18.0.1:${constants.DOCKER_PROXY_PORT}`);
await util.promisify(gHttpServer.listen.bind(gHttpServer))(constants.DOCKER_PROXY_PORT, '172.18.0.1'); await util.promisify(gHttpServer.listen.bind(gHttpServer))(constants.DOCKER_PROXY_PORT, '172.18.0.1');
} }

10
src/domains.js
View File

@@ -4,7 +4,7 @@ import constants from './constants.js';
import crypto from 'node:crypto'; import crypto from 'node:crypto';
import dashboard from './dashboard.js'; import dashboard from './dashboard.js';
import database from './database.js'; import database from './database.js';
import debugModule from 'debug'; import logger from './logger.js';
import eventlog from './eventlog.js'; import eventlog from './eventlog.js';
import mailServer from './mailserver.js'; import mailServer from './mailserver.js';
import notifications from './notifications.js'; import notifications from './notifications.js';
@@ -36,7 +36,7 @@ import dnsManual from './dns/manual.js';
import dnsPorkbun from './dns/porkbun.js'; import dnsPorkbun from './dns/porkbun.js';
import dnsWildcard from './dns/wildcard.js'; import dnsWildcard from './dns/wildcard.js';
const debug = debugModule('box:domains'); const { log, trace } = logger('domains');
const DOMAINS_FIELDS = [ 'domain', 'zoneName', 'provider', 'configJson', 'tlsConfigJson', 'wellKnownJson', 'fallbackCertificateJson' ].join(','); const DOMAINS_FIELDS = [ 'domain', 'zoneName', 'provider', 'configJson', 'tlsConfigJson', 'wellKnownJson', 'fallbackCertificateJson' ].join(',');
@@ -187,7 +187,7 @@ async function add(domain, data, auditSource) {
await eventlog.add(eventlog.ACTION_DOMAIN_ADD, auditSource, { domain, zoneName, provider }); await eventlog.add(eventlog.ACTION_DOMAIN_ADD, auditSource, { domain, zoneName, provider });
safe(mailServer.onDomainAdded(domain), { debug }); // background safe(mailServer.onDomainAdded(domain), { debug: log }); // background
} }
async function get(domain) { async function get(domain) {
@@ -341,7 +341,7 @@ async function getDomainObjectMap() {
async function checkConfigs(auditSource) { async function checkConfigs(auditSource) {
assert.strictEqual(typeof auditSource, 'object'); assert.strictEqual(typeof auditSource, 'object');
debug(`checkConfig: validating domain configs`); log(`checkConfig: validating domain configs`);
for (const domainObject of await list()) { for (const domainObject of await list()) {
if (domainObject.provider === 'noop' || domainObject.provider === 'manual' || domainObject.provider === 'wildcard') { if (domainObject.provider === 'noop' || domainObject.provider === 'manual' || domainObject.provider === 'wildcard') {
@@ -366,7 +366,7 @@ async function checkConfigs(auditSource) {
errorMessage = `General error: ${error.message}`; errorMessage = `General error: ${error.message}`;
} }
debug(`checkConfig: ${domainObject.domain} is not configured properly`, error); log(`checkConfig: ${domainObject.domain} is not configured properly`, error);
await notifications.pin(notifications.TYPE_DOMAIN_CONFIG_CHECK_FAILED, `Domain ${domainObject.domain} is not configured properly`, await notifications.pin(notifications.TYPE_DOMAIN_CONFIG_CHECK_FAILED, `Domain ${domainObject.domain} is not configured properly`,
errorMessage, { context: domainObject.domain }); errorMessage, { context: domainObject.domain });

20
src/dyndns.js
View File

@@ -1,7 +1,7 @@
import apps from './apps.js'; import apps from './apps.js';
import assert from 'node:assert'; import assert from 'node:assert';
import dashboard from './dashboard.js'; import dashboard from './dashboard.js';
import debugModule from 'debug'; import logger from './logger.js';
import dns from './dns.js'; import dns from './dns.js';
import eventlog from './eventlog.js'; import eventlog from './eventlog.js';
import fs from 'node:fs'; import fs from 'node:fs';
@@ -11,7 +11,7 @@ import paths from './paths.js';
import safe from 'safetydance'; import safe from 'safetydance';
import tasks from './tasks.js'; import tasks from './tasks.js';
const debug = debugModule('box:dyndns'); const { log, trace } = logger('dyndns');
// FIXME: this races with apptask. can result in a conflict if apptask is doing some dns operation and this code changes entries // FIXME: this races with apptask. can result in a conflict if apptask is doing some dns operation and this code changes entries
@@ -26,11 +26,11 @@ async function refreshDns(auditSource) {
const ipv6Changed = ipv6 && info.ipv6 !== ipv6; // both should be RFC 5952 format const ipv6Changed = ipv6 && info.ipv6 !== ipv6; // both should be RFC 5952 format
if (!ipv4Changed && !ipv6Changed) { if (!ipv4Changed && !ipv6Changed) {
debug(`refreshDns: no change in IP. ipv4: ${ipv4} ipv6: ${ipv6}`); log(`refreshDns: no change in IP. ipv4: ${ipv4} ipv6: ${ipv6}`);
return; return;
} }
debug(`refreshDns: updating IP from ${info.ipv4} to ipv4: ${ipv4} (changed: ${ipv4Changed}) ipv6: ${ipv6} (changed: ${ipv6Changed})`); log(`refreshDns: updating IP from ${info.ipv4} to ipv4: ${ipv4} (changed: ${ipv4Changed}) ipv6: ${ipv6} (changed: ${ipv6Changed})`);
const taskId = await tasks.add(tasks.TASK_SYNC_DYNDNS, [ ipv4Changed ? ipv4 : null, ipv6Changed ? ipv6 : null, auditSource ]); const taskId = await tasks.add(tasks.TASK_SYNC_DYNDNS, [ ipv4Changed ? ipv4 : null, ipv6Changed ? ipv6 : null, auditSource ]);
// background // background
@@ -57,15 +57,15 @@ async function sync(ipv4, ipv6, auditSource, progressCallback) {
let percent = 5; let percent = 5;
const { domain:dashboardDomain, fqdn:dashboardFqdn, subdomain:dashboardSubdomain } = await dashboard.getLocation(); const { domain:dashboardDomain, fqdn:dashboardFqdn, subdomain:dashboardSubdomain } = await dashboard.getLocation();
progressCallback({ percent, message: `Updating dashboard location ${dashboardFqdn}`}); progressCallback({ percent, message: `Updating dashboard location ${dashboardFqdn}`});
if (ipv4) await safe(dns.upsertDnsRecords(dashboardSubdomain, dashboardDomain, 'A', [ ipv4 ]), { debug }); if (ipv4) await safe(dns.upsertDnsRecords(dashboardSubdomain, dashboardDomain, 'A', [ ipv4 ]), { debug: log });
if (ipv6) await safe(dns.upsertDnsRecords(dashboardSubdomain, dashboardDomain, 'AAAA', [ ipv6 ]), { debug }); if (ipv6) await safe(dns.upsertDnsRecords(dashboardSubdomain, dashboardDomain, 'AAAA', [ ipv6 ]), { debug: log });
const { domain:mailDomain, fqdn:mailFqdn, subdomain:mailSubdomain } = await mailServer.getLocation(); const { domain:mailDomain, fqdn:mailFqdn, subdomain:mailSubdomain } = await mailServer.getLocation();
percent += 10; percent += 10;
progressCallback({ percent, message: `Updating mail location ${mailFqdn}`}); progressCallback({ percent, message: `Updating mail location ${mailFqdn}`});
if (dashboardFqdn !== mailFqdn) { if (dashboardFqdn !== mailFqdn) {
if (ipv4) await safe(dns.upsertDnsRecords(mailSubdomain, mailDomain, 'A', [ ipv4 ]), { debug }); if (ipv4) await safe(dns.upsertDnsRecords(mailSubdomain, mailDomain, 'A', [ ipv4 ]), { debug: log });
if (ipv6) await safe(dns.upsertDnsRecords(mailSubdomain, mailDomain, 'AAAA', [ ipv6 ]), { debug }); if (ipv6) await safe(dns.upsertDnsRecords(mailSubdomain, mailDomain, 'AAAA', [ ipv6 ]), { debug: log });
} }
const result = await apps.list(); const result = await apps.list();
@@ -79,8 +79,8 @@ async function sync(ipv4, ipv6, auditSource, progressCallback) {
.concat(app.aliasDomains); .concat(app.aliasDomains);
for (const location of locations) { for (const location of locations) {
if (ipv4) await safe(dns.upsertDnsRecords(location.subdomain, location.domain, 'A', [ ipv4 ]), { debug }); if (ipv4) await safe(dns.upsertDnsRecords(location.subdomain, location.domain, 'A', [ ipv4 ]), { debug: log });
if (ipv6) await safe(dns.upsertDnsRecords(location.subdomain, location.domain, 'AAAA', [ ipv6 ], { debug })); if (ipv6) await safe(dns.upsertDnsRecords(location.subdomain, location.domain, 'AAAA', [ ipv6 ], { debug: log }));
} }
} }

6
src/eventlog.js
View File

@@ -1,12 +1,12 @@
import assert from 'node:assert'; import assert from 'node:assert';
import crypto from 'node:crypto'; import crypto from 'node:crypto';
import database from './database.js'; import database from './database.js';
import debugModule from 'debug'; import logger from './logger.js';
import mysql from 'mysql2'; import mysql from 'mysql2';
import notifications from './notifications.js'; import notifications from './notifications.js';
import safe from 'safetydance'; import safe from 'safetydance';
const debug = debugModule('box:eventlog'); const { log, trace } = logger('eventlog');
const ACTION_ACTIVATE = 'cloudron.activate'; const ACTION_ACTIVATE = 'cloudron.activate';
const ACTION_USER_LOGIN = 'user.login'; const ACTION_USER_LOGIN = 'user.login';
@@ -119,7 +119,7 @@ async function cleanup(options) {
assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof options, 'object');
const creationTime = options.creationTime; const creationTime = options.creationTime;
debug(`cleanup: pruning events. creationTime: ${creationTime.toString()}`); log(`cleanup: pruning events. creationTime: ${creationTime.toString()}`);
// only these actions are pruned // only these actions are pruned
const actions = [ const actions = [

78
src/externalldap.js
View File

@@ -3,7 +3,7 @@ import AuditSource from './auditsource.js';
import BoxError from './boxerror.js'; import BoxError from './boxerror.js';
import constants from './constants.js'; import constants from './constants.js';
import cron from './cron.js'; import cron from './cron.js';
import debugModule from 'debug'; import logger from './logger.js';
import eventlog from './eventlog.js'; import eventlog from './eventlog.js';
import groups from './groups.js'; import groups from './groups.js';
import ldap from 'ldapjs'; import ldap from 'ldapjs';
@@ -13,7 +13,7 @@ import tasks from './tasks.js';
import users from './users.js'; import users from './users.js';
import util from 'node:util'; import util from 'node:util';
const debug = debugModule('box:externalldap'); const { log, trace } = logger('externalldap');
function removePrivateFields(ldapConfig) { function removePrivateFields(ldapConfig) {
@@ -38,7 +38,7 @@ function translateUser(ldapConfig, ldapUser) {
}; };
if (!user.username || !user.email || !user.displayName) { if (!user.username || !user.email || !user.displayName) {
debug(`[Invalid LDAP user] username=${user.username} email=${user.email} displayName=${user.displayName}`); log(`[Invalid LDAP user] username=${user.username} email=${user.email} displayName=${user.displayName}`);
return null; return null;
} }
@@ -79,7 +79,7 @@ async function getClient(config, options) {
return await new Promise((resolve, reject) => { return await new Promise((resolve, reject) => {
// ensure we don't just crash // ensure we don't just crash
client.on('error', function (error) { // don't reject, we must have gotten a bind error client.on('error', function (error) { // don't reject, we must have gotten a bind error
debug('getClient: ExternalLdap client error:', error); log('getClient: ExternalLdap client error:', error);
}); });
// skip bind auth if none exist or if not wanted // skip bind auth if none exist or if not wanted
@@ -133,16 +133,16 @@ async function supportsPagination(client) {
const result = await clientSearch(client, '', searchOptions); const result = await clientSearch(client, '', searchOptions);
const controls = result.supportedControl; const controls = result.supportedControl;
if (!controls || !Array.isArray(controls)) { if (!controls || !Array.isArray(controls)) {
debug('supportsPagination: no supportedControl attribute returned'); log('supportsPagination: no supportedControl attribute returned');
return false; return false;
} }
if (!controls.includes(ldap.PagedResultsControl.OID)) { if (!controls.includes(ldap.PagedResultsControl.OID)) {
debug('supportsPagination: server does not support pagination. Available controls:', controls); log('supportsPagination: server does not support pagination. Available controls:', controls);
return false; return false;
} }
debug('supportsPagination: server supports pagination'); log('supportsPagination: server supports pagination');
return true; return true;
} }
@@ -150,7 +150,7 @@ async function ldapGetByDN(config, dn) {
assert.strictEqual(typeof config, 'object'); assert.strictEqual(typeof config, 'object');
assert.strictEqual(typeof dn, 'string'); assert.strictEqual(typeof dn, 'string');
debug(`ldapGetByDN: Get object at ${dn}`); log(`ldapGetByDN: Get object at ${dn}`);
const client = await getClient(config, { bind: true }); const client = await getClient(config, { bind: true });
const paged = await supportsPagination(client); const paged = await supportsPagination(client);
@@ -343,7 +343,7 @@ async function startSyncer() {
if (config.provider === 'noop') throw new BoxError(BoxError.BAD_STATE, 'not enabled'); if (config.provider === 'noop') throw new BoxError(BoxError.BAD_STATE, 'not enabled');
const taskId = await tasks.add(tasks.TASK_SYNC_EXTERNAL_LDAP, []); const taskId = await tasks.add(tasks.TASK_SYNC_EXTERNAL_LDAP, []);
safe(tasks.startTask(taskId, {}), { debug }); // background safe(tasks.startTask(taskId, {}), { debug: log }); // background
return taskId; return taskId;
} }
@@ -353,7 +353,7 @@ async function syncUsers(config, progressCallback) {
const ldapUsers = await ldapUserSearch(config, {}); const ldapUsers = await ldapUserSearch(config, {});
debug(`syncUsers: Found ${ldapUsers.length} users`); log(`syncUsers: Found ${ldapUsers.length} users`);
let percent = 10; let percent = 10;
const step = 28 / (ldapUsers.length + 1); const step = 28 / (ldapUsers.length + 1);
@@ -369,27 +369,27 @@ async function syncUsers(config, progressCallback) {
const user = await users.getByUsername(ldapUser.username); const user = await users.getByUsername(ldapUser.username);
if (!user) { if (!user) {
debug(`syncUsers: [adding user] username=${ldapUser.username} email=${ldapUser.email} displayName=${ldapUser.displayName}`); log(`syncUsers: [adding user] username=${ldapUser.username} email=${ldapUser.email} displayName=${ldapUser.displayName}`);
const [userAddError] = await safe(users.add(ldapUser.email, { username: ldapUser.username, password: null, displayName: ldapUser.displayName, source: 'ldap' }, AuditSource.EXTERNAL_LDAP)); const [userAddError] = await safe(users.add(ldapUser.email, { username: ldapUser.username, password: null, displayName: ldapUser.displayName, source: 'ldap' }, AuditSource.EXTERNAL_LDAP));
if (userAddError) debug('syncUsers: Failed to create user. %j %o', ldapUser, userAddError); if (userAddError) log('syncUsers: Failed to create user. %j %o', ldapUser, userAddError);
} else if (user.source !== 'ldap') { } else if (user.source !== 'ldap') {
debug(`syncUsers: [mapping user] username=${ldapUser.username} email=${ldapUser.email} displayName=${ldapUser.displayName}`); log(`syncUsers: [mapping user] username=${ldapUser.username} email=${ldapUser.email} displayName=${ldapUser.displayName}`);
const [userMappingError] = await safe(users.update(user, { email: ldapUser.email, fallbackEmail: ldapUser.email, displayName: ldapUser.displayName, source: 'ldap' }, AuditSource.EXTERNAL_LDAP)); const [userMappingError] = await safe(users.update(user, { email: ldapUser.email, fallbackEmail: ldapUser.email, displayName: ldapUser.displayName, source: 'ldap' }, AuditSource.EXTERNAL_LDAP));
if (userMappingError) debug('Failed to map user. %j %o', ldapUser, userMappingError); if (userMappingError) log('Failed to map user. %j %o', ldapUser, userMappingError);
} else if (user.email !== ldapUser.email || user.displayName !== ldapUser.displayName) { } else if (user.email !== ldapUser.email || user.displayName !== ldapUser.displayName) {
debug(`syncUsers: [updating user] username=${ldapUser.username} email=${ldapUser.email} displayName=${ldapUser.displayName}`); log(`syncUsers: [updating user] username=${ldapUser.username} email=${ldapUser.email} displayName=${ldapUser.displayName}`);
const [userUpdateError] = await safe(users.update(user, { email: ldapUser.email, fallbackEmail: ldapUser.email, displayName: ldapUser.displayName }, AuditSource.EXTERNAL_LDAP)); const [userUpdateError] = await safe(users.update(user, { email: ldapUser.email, fallbackEmail: ldapUser.email, displayName: ldapUser.displayName }, AuditSource.EXTERNAL_LDAP));
if (userUpdateError) debug('Failed to update user. %j %o', ldapUser, userUpdateError); if (userUpdateError) log('Failed to update user. %j %o', ldapUser, userUpdateError);
} else { } else {
// user known and up-to-date // user known and up-to-date
debug(`syncUsers: [up-to-date user] username=${ldapUser.username} email=${ldapUser.email} displayName=${ldapUser.displayName}`); log(`syncUsers: [up-to-date user] username=${ldapUser.username} email=${ldapUser.email} displayName=${ldapUser.displayName}`);
} }
} }
debug('syncUsers: done'); log('syncUsers: done');
} }
async function syncGroups(config, progressCallback) { async function syncGroups(config, progressCallback) {
@@ -397,15 +397,15 @@ async function syncGroups(config, progressCallback) {
assert.strictEqual(typeof progressCallback, 'function'); assert.strictEqual(typeof progressCallback, 'function');
if (!config.syncGroups) { if (!config.syncGroups) {
debug('syncGroups: Group sync is disabled'); log('syncGroups: Group sync is disabled');
progressCallback({ percent: 70, message: 'Skipping group sync' }); progressCallback({ percent: 70, message: 'Skipping group sync' });
return []; return [];
} }
const ldapGroups = await ldapGroupSearch(config, {}); const ldapGroups = await ldapGroupSearch(config, {});
debug(`syncGroups: Found ${ldapGroups.length} groups:`); log(`syncGroups: Found ${ldapGroups.length} groups:`);
debug(ldapGroups); log(ldapGroups);
let percent = 40; let percent = 40;
const step = 28 / (ldapGroups.length + 1); const step = 28 / (ldapGroups.length + 1);
@@ -423,19 +423,19 @@ async function syncGroups(config, progressCallback) {
const result = await groups.getByName(groupName); const result = await groups.getByName(groupName);
if (!result) { if (!result) {
debug(`syncGroups: [adding group] groupname=${groupName}`); log(`syncGroups: [adding group] groupname=${groupName}`);
const [error] = await safe(groups.add({ name: groupName, source: 'ldap' }, AuditSource.EXTERNAL_LDAP)); const [error] = await safe(groups.add({ name: groupName, source: 'ldap' }, AuditSource.EXTERNAL_LDAP));
if (error) debug('syncGroups: Failed to create group', groupName, error); if (error) log('syncGroups: Failed to create group', groupName, error);
} else { } else {
// convert local group to ldap group. 2 reasons: // convert local group to ldap group. 2 reasons:
// 1. we reset source flag when externalldap is disabled. if we renable, it automatically converts // 1. we reset source flag when externalldap is disabled. if we renable, it automatically converts
// 2. externalldap connector usually implies user wants to user external users/groups. // 2. externalldap connector usually implies user wants to user external users/groups.
groups.update(result.id, { source: 'ldap' }); groups.update(result.id, { source: 'ldap' });
debug(`syncGroups: [up-to-date group] groupname=${groupName}`); log(`syncGroups: [up-to-date group] groupname=${groupName}`);
} }
} }
debug('syncGroups: sync done'); log('syncGroups: sync done');
} }
async function syncGroupMembers(config, progressCallback) { async function syncGroupMembers(config, progressCallback) {
@@ -443,14 +443,14 @@ async function syncGroupMembers(config, progressCallback) {
assert.strictEqual(typeof progressCallback, 'function'); assert.strictEqual(typeof progressCallback, 'function');
if (!config.syncGroups) { if (!config.syncGroups) {
debug('syncGroupMembers: Group users sync is disabled'); log('syncGroupMembers: Group users sync is disabled');
progressCallback({ percent: 98, message: 'Skipping group member sync' }); progressCallback({ percent: 98, message: 'Skipping group member sync' });
return []; return [];
} }
const allGroups = await groups.listWithMembers(); const allGroups = await groups.listWithMembers();
const ldapGroups = allGroups.filter(function (g) { return g.source === 'ldap'; }); const ldapGroups = allGroups.filter(function (g) { return g.source === 'ldap'; });
debug(`syncGroupMembers: Found ${ldapGroups.length} groups to sync users`); log(`syncGroupMembers: Found ${ldapGroups.length} groups to sync users`);
let percent = 70; let percent = 70;
const step = 28 / (ldapGroups.length + 1); const step = 28 / (ldapGroups.length + 1);
@@ -458,11 +458,11 @@ async function syncGroupMembers(config, progressCallback) {
for (const ldapGroup of ldapGroups) { for (const ldapGroup of ldapGroups) {
percent = Math.min(percent + step, 98); percent = Math.min(percent + step, 98);
progressCallback({ percent: Math.round(percent), message: `Syncing members of ${ldapGroup.name}` }); progressCallback({ percent: Math.round(percent), message: `Syncing members of ${ldapGroup.name}` });
debug(`syncGroupMembers: Sync users for group ${ldapGroup.name}`); log(`syncGroupMembers: Sync users for group ${ldapGroup.name}`);
const result = await ldapGroupSearch(config, {}); const result = await ldapGroupSearch(config, {});
if (!result || result.length === 0) { if (!result || result.length === 0) {
debug(`syncGroupMembers: Unable to find group ${ldapGroup.name} ignoring for now.`); log(`syncGroupMembers: Unable to find group ${ldapGroup.name} ignoring for now.`);
continue; continue;
} }
@@ -473,7 +473,7 @@ async function syncGroupMembers(config, progressCallback) {
}); });
if (!found) { if (!found) {
debug(`syncGroupMembers: Unable to find group ${ldapGroup.name} ignoring for now.`); log(`syncGroupMembers: Unable to find group ${ldapGroup.name} ignoring for now.`);
continue; continue;
} }
@@ -482,24 +482,24 @@ async function syncGroupMembers(config, progressCallback) {
// if only one entry is in the group ldap returns a string, not an array! // if only one entry is in the group ldap returns a string, not an array!
if (typeof ldapGroupMembers === 'string') ldapGroupMembers = [ ldapGroupMembers ]; if (typeof ldapGroupMembers === 'string') ldapGroupMembers = [ ldapGroupMembers ];
debug(`syncGroupMembers: Group ${ldapGroup.name} has ${ldapGroupMembers.length} members.`); log(`syncGroupMembers: Group ${ldapGroup.name} has ${ldapGroupMembers.length} members.`);
const userIds = []; const userIds = [];
for (const memberDn of ldapGroupMembers) { for (const memberDn of ldapGroupMembers) {
const [ldapError, memberResult] = await safe(ldapGetByDN(config, memberDn)); const [ldapError, memberResult] = await safe(ldapGetByDN(config, memberDn));
if (ldapError) { if (ldapError) {
debug(`syncGroupMembers: Group ${ldapGroup.name} failed to get ${memberDn}: %o`, ldapError); log(`syncGroupMembers: Group ${ldapGroup.name} failed to get ${memberDn}: %o`, ldapError);
continue; continue;
} }
debug(`syncGroupMembers: Group ${ldapGroup.name} has member object ${memberDn}`); log(`syncGroupMembers: Group ${ldapGroup.name} has member object ${memberDn}`);
const username = memberResult[config.usernameField]?.toLowerCase(); const username = memberResult[config.usernameField]?.toLowerCase();
if (!username) continue; if (!username) continue;
const [getError, userObject] = await safe(users.getByUsername(username)); const [getError, userObject] = await safe(users.getByUsername(username));
if (getError || !userObject) { if (getError || !userObject) {
debug(`syncGroupMembers: Failed to get user by username ${username}. %o`, getError ? getError : 'User not found'); log(`syncGroupMembers: Failed to get user by username ${username}. %o`, getError ? getError : 'User not found');
continue; continue;
} }
@@ -507,15 +507,15 @@ async function syncGroupMembers(config, progressCallback) {
} }
const membersChanged = ldapGroup.userIds.length !== userIds.length || ldapGroup.userIds.some(id => !userIds.includes(id)); const membersChanged = ldapGroup.userIds.length !== userIds.length || ldapGroup.userIds.some(id => !userIds.includes(id));
if (membersChanged) { if (membersChanged) {
debug(`syncGroupMembers: Group ${ldapGroup.name} changed.`); log(`syncGroupMembers: Group ${ldapGroup.name} changed.`);
const [setError] = await safe(groups.setMembers(ldapGroup, userIds, { skipSourceCheck: true }, AuditSource.EXTERNAL_LDAP)); const [setError] = await safe(groups.setMembers(ldapGroup, userIds, { skipSourceCheck: true }, AuditSource.EXTERNAL_LDAP));
if (setError) debug(`syncGroupMembers: Failed to set members of group ${ldapGroup.name}. %o`, setError); if (setError) log(`syncGroupMembers: Failed to set members of group ${ldapGroup.name}. %o`, setError);
} else { } else {
debug(`syncGroupMembers: Group ${ldapGroup.name} is unchanged.`); log(`syncGroupMembers: Group ${ldapGroup.name} is unchanged.`);
} }
} }
debug('syncGroupMembers: done'); log('syncGroupMembers: done');
} }
async function sync(progressCallback) { async function sync(progressCallback) {
@@ -532,7 +532,7 @@ async function sync(progressCallback) {
progressCallback({ percent: 100, message: 'Done' }); progressCallback({ percent: 100, message: 'Done' });
debug('sync: done'); log('sync: done');
} }
export default { export default {

6
src/hush.js
View File

@@ -1,10 +1,10 @@
import assert from 'node:assert'; import assert from 'node:assert';
import BoxError from './boxerror.js'; import BoxError from './boxerror.js';
import crypto from 'node:crypto'; import crypto from 'node:crypto';
import debugModule from 'debug'; import logger from './logger.js';
import { Transform as TransformStream } from 'node:stream'; import { Transform as TransformStream } from 'node:stream';
const debug = debugModule('box:hush'); const { log, trace } = logger('hush');
class EncryptStream extends TransformStream { class EncryptStream extends TransformStream {
constructor(encryption) { constructor(encryption) {
@@ -144,7 +144,7 @@ function decryptFilePath(filePath, encryption) {
decryptedParts.push(plainTextString); decryptedParts.push(plainTextString);
} catch (error) { } catch (error) {
debug(`Error decrypting part ${part} of path ${filePath}: %o`, error); log(`Error decrypting part ${part} of path ${filePath}: %o`, error);
return { error: new BoxError(BoxError.CRYPTO_ERROR, `Decryption error. ${part} of path ${filePath}: ${error.message}`) }; return { error: new BoxError(BoxError.CRYPTO_ERROR, `Decryption error. ${part} of path ${filePath}: ${error.message}`) };
} }
} }

18
src/janitor.js
View File

@@ -1,22 +1,22 @@
import assert from 'node:assert'; import assert from 'node:assert';
import BoxError from './boxerror.js'; import BoxError from './boxerror.js';
import debugModule from 'debug'; import logger from './logger.js';
import Docker from 'dockerode'; import Docker from 'dockerode';
import safe from 'safetydance'; import safe from 'safetydance';
import tokens from './tokens.js'; import tokens from './tokens.js';
const debug = debugModule('box:janitor'); const { log, trace } = logger('janitor');
const gConnection = new Docker({ socketPath: '/var/run/docker.sock' }); const gConnection = new Docker({ socketPath: '/var/run/docker.sock' });
async function cleanupTokens() { async function cleanupTokens() {
debug('Cleaning up expired tokens'); log('Cleaning up expired tokens');
const [error, result] = await safe(tokens.delExpired()); const [error, result] = await safe(tokens.delExpired());
if (error) return debug('cleanupTokens: error removing expired tokens. %o', error); if (error) return log('cleanupTokens: error removing expired tokens. %o', error);
debug(`Cleaned up ${result} expired tokens`); log(`Cleaned up ${result} expired tokens`);
} }
async function cleanupTmpVolume(containerInfo) { async function cleanupTmpVolume(containerInfo) {
@@ -24,7 +24,7 @@ async function cleanupTmpVolume(containerInfo) {
const cmd = 'find /tmp -type f -mtime +10 -exec rm -rf {} +'.split(' '); // 10 day old files const cmd = 'find /tmp -type f -mtime +10 -exec rm -rf {} +'.split(' '); // 10 day old files
debug(`cleanupTmpVolume ${JSON.stringify(containerInfo.Names)}`); log(`cleanupTmpVolume ${JSON.stringify(containerInfo.Names)}`);
const [error, execContainer] = await safe(gConnection.getContainer(containerInfo.Id).exec({ Cmd: cmd, AttachStdout: true, AttachStderr: true, Tty: false })); const [error, execContainer] = await safe(gConnection.getContainer(containerInfo.Id).exec({ Cmd: cmd, AttachStdout: true, AttachStderr: true, Tty: false }));
if (error) throw new BoxError(BoxError.DOCKER_ERROR, `Failed to exec container: ${error.message}`); if (error) throw new BoxError(BoxError.DOCKER_ERROR, `Failed to exec container: ${error.message}`);
@@ -41,16 +41,16 @@ async function cleanupTmpVolume(containerInfo) {
} }
async function cleanupDockerVolumes() { async function cleanupDockerVolumes() {
debug('Cleaning up docker volumes'); log('Cleaning up docker volumes');
const [error, containers] = await safe(gConnection.listContainers({ all: 0 })); const [error, containers] = await safe(gConnection.listContainers({ all: 0 }));
if (error) throw new BoxError(BoxError.DOCKER_ERROR, error); if (error) throw new BoxError(BoxError.DOCKER_ERROR, error);
for (const container of containers) { for (const container of containers) {
await safe(cleanupTmpVolume(container), { debug }); // intentionally ignore error await safe(cleanupTmpVolume(container), { debug: log }); // intentionally ignore error
} }
debug('Cleaned up docker volumes'); log('Cleaned up docker volumes');
} }
export default { export default {

46
src/ldapserver.js
View File

@@ -4,7 +4,7 @@ import apps from './apps.js';
import AuditSource from './auditsource.js'; import AuditSource from './auditsource.js';
import BoxError from './boxerror.js'; import BoxError from './boxerror.js';
import constants from './constants.js'; import constants from './constants.js';
import debugModule from 'debug'; import logger from './logger.js';
import eventlog from './eventlog.js'; import eventlog from './eventlog.js';
import groups from './groups.js'; import groups from './groups.js';
import ldap from 'ldapjs'; import ldap from 'ldapjs';
@@ -13,7 +13,7 @@ import safe from 'safetydance';
import users from './users.js'; import users from './users.js';
import util from 'node:util'; import util from 'node:util';
const debug = debugModule('box:ldapserver'); const { log, trace } = logger('ldapserver');
let _MOCK_APP = null; let _MOCK_APP = null;
@@ -142,7 +142,7 @@ function finalSend(results, req, res, next) {
} }
async function userSearch(req, res, next) { async function userSearch(req, res, next) {
debug('user search: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id); log('user search: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id);
const [error, result] = await safe(getUsersWithAccessToApp(req)); const [error, result] = await safe(getUsersWithAccessToApp(req));
if (error) return next(new ldap.OperationsError(error.message)); if (error) return next(new ldap.OperationsError(error.message));
@@ -195,7 +195,7 @@ async function userSearch(req, res, next) {
} }
async function groupSearch(req, res, next) { async function groupSearch(req, res, next) {
debug('group search: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id); log('group search: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id);
const results = []; const results = [];
@@ -230,7 +230,7 @@ async function groupSearch(req, res, next) {
} }
async function groupUsersCompare(req, res, next) { async function groupUsersCompare(req, res, next) {
debug('group users compare: dn %s, attribute %s, value %s (from %s)', req.dn.toString(), req.attribute, req.value, req.connection.ldap.id); log('group users compare: dn %s, attribute %s, value %s (from %s)', req.dn.toString(), req.attribute, req.value, req.connection.ldap.id);
const [error, result] = await safe(getUsersWithAccessToApp(req)); const [error, result] = await safe(getUsersWithAccessToApp(req));
if (error) return next(new ldap.OperationsError(error.message)); if (error) return next(new ldap.OperationsError(error.message));
@@ -245,7 +245,7 @@ async function groupUsersCompare(req, res, next) {
} }
async function groupAdminsCompare(req, res, next) { async function groupAdminsCompare(req, res, next) {
debug('group admins compare: dn %s, attribute %s, value %s (from %s)', req.dn.toString(), req.attribute, req.value, req.connection.ldap.id); log('group admins compare: dn %s, attribute %s, value %s (from %s)', req.dn.toString(), req.attribute, req.value, req.connection.ldap.id);
const [error, result] = await safe(getUsersWithAccessToApp(req)); const [error, result] = await safe(getUsersWithAccessToApp(req));
if (error) return next(new ldap.OperationsError(error.message)); if (error) return next(new ldap.OperationsError(error.message));
@@ -260,7 +260,7 @@ async function groupAdminsCompare(req, res, next) {
} }
async function mailboxSearch(req, res, next) { async function mailboxSearch(req, res, next) {
debug('mailbox search: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id); log('mailbox search: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id);
// if cn is set OR filter is mail= we only search for one mailbox specifically // if cn is set OR filter is mail= we only search for one mailbox specifically
let email, dn; let email, dn;
@@ -350,7 +350,7 @@ async function mailboxSearch(req, res, next) {
} }
async function mailAliasSearch(req, res, next) { async function mailAliasSearch(req, res, next) {
debug('mail alias get: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id); log('mail alias get: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id);
if (!req.dn.rdns[0].attrs.cn) return next(new ldap.NoSuchObjectError('Missing CN')); if (!req.dn.rdns[0].attrs.cn) return next(new ldap.NoSuchObjectError('Missing CN'));
@@ -389,7 +389,7 @@ async function mailAliasSearch(req, res, next) {
} }
async function mailingListSearch(req, res, next) { async function mailingListSearch(req, res, next) {
debug('mailing list get: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id); log('mailing list get: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id);
if (!req.dn.rdns[0].attrs.cn) return next(new ldap.NoSuchObjectError('Missing CN')); if (!req.dn.rdns[0].attrs.cn) return next(new ldap.NoSuchObjectError('Missing CN'));
@@ -433,7 +433,7 @@ async function mailingListSearch(req, res, next) {
// Will attach req.user if successful // Will attach req.user if successful
async function authenticateUser(req, res, next) { async function authenticateUser(req, res, next) {
debug('user bind: %s (from %s)', req.dn.toString(), req.connection.ldap.id); log('user bind: %s (from %s)', req.dn.toString(), req.connection.ldap.id);
const appId = req.app.id; const appId = req.app.id;
@@ -478,7 +478,7 @@ async function verifyMailboxPassword(mailbox, password) {
} }
async function authenticateSftp(req, res, next) { async function authenticateSftp(req, res, next) {
debug('sftp auth: %s (from %s)', req.dn.toString(), req.connection.ldap.id); log('sftp auth: %s (from %s)', req.dn.toString(), req.connection.ldap.id);
if (!req.dn.rdns[0].attrs.cn) return next(new ldap.NoSuchObjectError('Missing CN')); if (!req.dn.rdns[0].attrs.cn) return next(new ldap.NoSuchObjectError('Missing CN'));
@@ -492,13 +492,13 @@ async function authenticateSftp(req, res, next) {
const [verifyError] = await safe(users.verifyWithUsername(parts[0], req.credentials, app.id, { skipTotpCheck: true })); const [verifyError] = await safe(users.verifyWithUsername(parts[0], req.credentials, app.id, { skipTotpCheck: true }));
if (verifyError) return next(new ldap.InvalidCredentialsError(verifyError.message)); if (verifyError) return next(new ldap.InvalidCredentialsError(verifyError.message));
debug('sftp auth: success'); log('sftp auth: success');
res.end(); res.end();
} }
async function userSearchSftp(req, res, next) { async function userSearchSftp(req, res, next) {
debug('sftp user search: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id); log('sftp user search: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id);
if (req.filter.attribute !== 'username' || !req.filter.value) return next(new ldap.NoSuchObjectError()); if (req.filter.attribute !== 'username' || !req.filter.value) return next(new ldap.NoSuchObjectError());
@@ -556,7 +556,7 @@ async function verifyAppMailboxPassword(serviceId, username, password) {
} }
async function authenticateService(serviceId, dn, req, res, next) { async function authenticateService(serviceId, dn, req, res, next) {
debug(`authenticateService: ${req.dn.toString()} (from ${req.connection.ldap.id})`); log(`authenticateService: ${req.dn.toString()} (from ${req.connection.ldap.id})`);
if (!dn.rdns[0].attrs.cn) return next(new ldap.NoSuchObjectError(dn.toString())); if (!dn.rdns[0].attrs.cn) return next(new ldap.NoSuchObjectError(dn.toString()));
@@ -608,7 +608,7 @@ async function authenticateMail(req, res, next) {
// https://ldapwiki.com/wiki/RootDSE / RFC 4512 - ldapsearch -x -h "${CLOUDRON_LDAP_SERVER}" -p "${CLOUDRON_LDAP_PORT}" -b "" -s base // https://ldapwiki.com/wiki/RootDSE / RFC 4512 - ldapsearch -x -h "${CLOUDRON_LDAP_SERVER}" -p "${CLOUDRON_LDAP_PORT}" -b "" -s base
// ldapjs seems to call this handler for everything when search === '' // ldapjs seems to call this handler for everything when search === ''
async function maybeRootDSE(req, res, next) { async function maybeRootDSE(req, res, next) {
debug(`maybeRootDSE: requested with scope:${req.scope} dn:${req.dn.toString()}`); log(`maybeRootDSE: requested with scope:${req.scope} dn:${req.dn.toString()}`);
if (req.scope !== 'base') return next(new ldap.NoSuchObjectError()); // per the spec, rootDSE search require base scope if (req.scope !== 'base') return next(new ldap.NoSuchObjectError()); // per the spec, rootDSE search require base scope
if (!req.dn || req.dn.toString() !== '') return next(new ldap.NoSuchObjectError()); if (!req.dn || req.dn.toString() !== '') return next(new ldap.NoSuchObjectError());
@@ -633,16 +633,16 @@ async function start() {
const logger = { const logger = {
trace: NOOP, trace: NOOP,
debug: NOOP, debug: NOOP,
info: debug, info: log,
warn: debug, warn: log,
error: debug, error: log,
fatal: debug fatal: log
}; };
gServer = ldap.createServer({ log: logger }); gServer = ldap.createServer({ log: logger });
gServer.on('error', function (error) { gServer.on('error', function (error) {
debug('start: server error. %o', error); log('start: server error. %o', error);
}); });
gServer.search('ou=users,dc=cloudron', authenticateApp, userSearch); gServer.search('ou=users,dc=cloudron', authenticateApp, userSearch);
@@ -670,14 +670,14 @@ async function start() {
// this is the bind for addons (after bind, they might search and authenticate) // this is the bind for addons (after bind, they might search and authenticate)
gServer.bind('ou=addons,dc=cloudron', function(req, res /*, next */) { gServer.bind('ou=addons,dc=cloudron', function(req, res /*, next */) {
debug('addons bind: %s', req.dn.toString()); // note: cn can be email or id log('addons bind: %s', req.dn.toString()); // note: cn can be email or id
res.end(); res.end();
}); });
// this is the bind for apps (after bind, they might search and authenticate user) // this is the bind for apps (after bind, they might search and authenticate user)
gServer.bind('ou=apps,dc=cloudron', function(req, res /*, next */) { gServer.bind('ou=apps,dc=cloudron', function(req, res /*, next */) {
// TODO: validate password // TODO: validate password
debug('application bind: %s', req.dn.toString()); log('application bind: %s', req.dn.toString());
res.end(); res.end();
}); });
@@ -693,7 +693,7 @@ async function start() {
// just log that an attempt was made to unknown route, this helps a lot during app packaging // just log that an attempt was made to unknown route, this helps a lot during app packaging
gServer.use(function(req, res, next) { gServer.use(function(req, res, next) {
debug('not handled: dn %s, scope %s, filter %s (from %s)', req.dn ? req.dn.toString() : '-', req.scope, req.filter ? req.filter.toString() : '-', req.connection.ldap.id); log('not handled: dn %s, scope %s, filter %s (from %s)', req.dn ? req.dn.toString() : '-', req.scope, req.filter ? req.filter.toString() : '-', req.connection.ldap.id);
return next(); return next();
}); });

24
src/locks.js
View File

@@ -1,10 +1,10 @@
import assert from 'node:assert'; import assert from 'node:assert';
import BoxError from './boxerror.js'; import BoxError from './boxerror.js';
import database from './database.js'; import database from './database.js';
import debugModule from 'debug'; import logger from './logger.js';
import promiseRetry from './promise-retry.js'; import promiseRetry from './promise-retry.js';
const debug = debugModule('box:locks'); const { log, trace } = logger('locks');
const TYPE_APP_TASK_PREFIX = 'app_task_'; const TYPE_APP_TASK_PREFIX = 'app_task_';
const TYPE_APP_BACKUP_PREFIX = 'app_backup_'; const TYPE_APP_BACKUP_PREFIX = 'app_backup_';
@@ -31,7 +31,7 @@ async function write(value) {
const result = await database.query('UPDATE locks SET dataJson=?, version=version+1 WHERE id=? AND version=?', [ JSON.stringify(value.data), 'platform', value.version ]); const result = await database.query('UPDATE locks SET dataJson=?, version=version+1 WHERE id=? AND version=?', [ JSON.stringify(value.data), 'platform', value.version ]);
if (result.affectedRows !== 1) throw new BoxError(BoxError.CONFLICT, 'Someone updated before we did'); if (result.affectedRows !== 1) throw new BoxError(BoxError.CONFLICT, 'Someone updated before we did');
debug(`write: current locks: ${JSON.stringify(value.data)}`); log(`write: current locks: ${JSON.stringify(value.data)}`);
} }
function canAcquire(data, type) { function canAcquire(data, type) {
@@ -59,58 +59,58 @@ function canAcquire(data, type) {
async function acquire(type) { async function acquire(type) {
assert.strictEqual(typeof type, 'string'); assert.strictEqual(typeof type, 'string');
await promiseRetry({ times: Number.MAX_SAFE_INTEGER, interval: 100, debug, retry: (error) => error.reason === BoxError.CONFLICT }, async () => { await promiseRetry({ times: Number.MAX_SAFE_INTEGER, interval: 100, debug: log, retry: (error) => error.reason === BoxError.CONFLICT }, async () => {
const { version, data } = await read(); const { version, data } = await read();
const error = canAcquire(data, type); const error = canAcquire(data, type);
if (error) throw error; if (error) throw error;
data[type] = gTaskId; data[type] = gTaskId;
await write({ version, data }); await write({ version, data });
debug(`acquire: ${type}`); log(`acquire: ${type}`);
}); });
} }
async function wait(type) { async function wait(type) {
assert.strictEqual(typeof type, 'string'); assert.strictEqual(typeof type, 'string');
await promiseRetry({ times: Number.MAX_SAFE_INTEGER, interval: 10000, debug }, async () => await acquire(type)); await promiseRetry({ times: Number.MAX_SAFE_INTEGER, interval: 10000, debug: log }, async () => await acquire(type));
} }
async function release(type) { async function release(type) {
assert.strictEqual(typeof type, 'string'); assert.strictEqual(typeof type, 'string');
await promiseRetry({ times: Number.MAX_SAFE_INTEGER, interval: 100, debug, retry: (error) => error.reason === BoxError.CONFLICT }, async () => { await promiseRetry({ times: Number.MAX_SAFE_INTEGER, interval: 100, debug: log, retry: (error) => error.reason === BoxError.CONFLICT }, async () => {
const { version, data } = await read(); const { version, data } = await read();
if (!(type in data)) throw new BoxError(BoxError.BAD_STATE, `Lock ${type} was never acquired`); if (!(type in data)) throw new BoxError(BoxError.BAD_STATE, `Lock ${type} was never acquired`);
if (data[type] !== gTaskId) throw new BoxError(BoxError.BAD_STATE, `Task ${gTaskId} attempted to release lock ${type} acquired by ${data[type]}`); if (data[type] !== gTaskId) throw new BoxError(BoxError.BAD_STATE, `Task ${gTaskId} attempted to release lock ${type} acquired by ${data[type]}`);
delete data[type]; delete data[type];
await write({ version, data }); await write({ version, data });
debug(`release: ${type}`); log(`release: ${type}`);
}); });
} }
async function releaseAll() { async function releaseAll() {
await database.query('DELETE FROM locks'); await database.query('DELETE FROM locks');
await database.query('INSERT INTO locks (id, dataJson) VALUES (?, ?)', [ 'platform', JSON.stringify({}) ]); await database.query('INSERT INTO locks (id, dataJson) VALUES (?, ?)', [ 'platform', JSON.stringify({}) ]);
debug('releaseAll: all locks released'); log('releaseAll: all locks released');
} }
// identify programming errors in tasks that forgot to clean up locks // identify programming errors in tasks that forgot to clean up locks
async function releaseByTaskId(taskId) { async function releaseByTaskId(taskId) {
assert.strictEqual(typeof taskId, 'string'); assert.strictEqual(typeof taskId, 'string');
await promiseRetry({ times: Number.MAX_SAFE_INTEGER, interval: 100, debug, retry: (error) => error.reason === BoxError.CONFLICT }, async () => { await promiseRetry({ times: Number.MAX_SAFE_INTEGER, interval: 100, debug: log, retry: (error) => error.reason === BoxError.CONFLICT }, async () => {
const { version, data } = await read(); const { version, data } = await read();
for (const type of Object.keys(data)) { for (const type of Object.keys(data)) {
if (data[type] === taskId) { if (data[type] === taskId) {
debug(`releaseByTaskId: task ${taskId} forgot to unlock ${type}`); log(`releaseByTaskId: task ${taskId} forgot to unlock ${type}`);
delete data[type]; delete data[type];
} }
} }
await write({ version, data }); await write({ version, data });
debug(`releaseByTaskId: ${taskId}`); log(`releaseByTaskId: ${taskId}`);
}); });
} }

16
src/logger.js Normal file
View File

@@ -0,0 +1,16 @@
import util from 'node:util';
const TRACE_ENABLED = false;
const LOG_ENABLED = process.env.BOX_ENV !== 'test' || !!process.env.LOG;
function output(namespace, args) {
const ts = new Date().toISOString();
process.stdout.write(`${ts} ${namespace}: ${util.format(...args)}\n`);
}
export default function logger(namespace) {
return {
log: LOG_ENABLED ? (...args) => output(namespace, args) : () => {},
trace: TRACE_ENABLED ? (...args) => output(namespace, args) : () => {},
};
}

6
src/logs.js
View File

@@ -1,11 +1,11 @@
import assert from 'node:assert'; import assert from 'node:assert';
import child_process from 'node:child_process'; import child_process from 'node:child_process';
import debugModule from 'debug'; import logger from './logger.js';
import path from 'node:path'; import path from 'node:path';
import stream from 'node:stream'; import stream from 'node:stream';
import { StringDecoder } from 'node:string_decoder'; import { StringDecoder } from 'node:string_decoder';
const debug = debugModule('box:logs'); const { log, trace } = logger('logs');
const TransformStream = stream.Transform; const TransformStream = stream.Transform;
const LOGTAIL_CMD = path.join(import.meta.dirname, 'scripts/logtail.sh'); const LOGTAIL_CMD = path.join(import.meta.dirname, 'scripts/logtail.sh');
@@ -74,7 +74,7 @@ function tail(filePaths, options) {
cp.terminate = () => { cp.terminate = () => {
child_process.execFile('/usr/bin/sudo', [ KILL_CHILD_CMD, cp.pid, process.pid ], { encoding: 'utf8' }, (error, stdout, stderr) => { child_process.execFile('/usr/bin/sudo', [ KILL_CHILD_CMD, cp.pid, process.pid ], { encoding: 'utf8' }, (error, stdout, stderr) => {
if (error) debug(`tail: failed to kill children`, stdout, stderr); if (error) log(`tail: failed to kill children`, stdout, stderr);
}); });
}; };

36
src/mail.js
View File

@@ -2,7 +2,7 @@ import assert from 'node:assert';
import BoxError from './boxerror.js'; import BoxError from './boxerror.js';
import constants from './constants.js'; import constants from './constants.js';
import database from './database.js'; import database from './database.js';
import debugModule from 'debug'; import logger from './logger.js';
import dig from './dig.js'; import dig from './dig.js';
import dns from './dns.js'; import dns from './dns.js';
import eventlog from './eventlog.js'; import eventlog from './eventlog.js';
@@ -22,7 +22,7 @@ import superagent from '@cloudron/superagent';
import validator from './validator.js'; import validator from './validator.js';
import _ from './underscore.js'; import _ from './underscore.js';
const debug = debugModule('box:mail'); const { log, trace } = logger('mail');
const shell = shellModule('mail'); const shell = shellModule('mail');
const OWNERTYPE_USER = 'user'; const OWNERTYPE_USER = 'user';
@@ -524,14 +524,14 @@ async function checkRbl(type, mailDomain) {
const [rblError, records] = await safe(dig.resolve(`${flippedIp}.${rblServer.dns}`, 'A', DNS_OPTIONS)); const [rblError, records] = await safe(dig.resolve(`${flippedIp}.${rblServer.dns}`, 'A', DNS_OPTIONS));
if (rblError || records.length === 0) continue; // not listed if (rblError || records.length === 0) continue; // not listed
debug(`checkRbl (${domain}) flippedIp: ${flippedIp} is in the blocklist of ${rblServer.dns}: ${JSON.stringify(records)}`); log(`checkRbl (${domain}) flippedIp: ${flippedIp} is in the blocklist of ${rblServer.dns}: ${JSON.stringify(records)}`);
const result = Object.assign({}, rblServer); const result = Object.assign({}, rblServer);
const [error2, txtRecords] = await safe(dig.resolve(`${flippedIp}.${rblServer.dns}`, 'TXT', DNS_OPTIONS)); const [error2, txtRecords] = await safe(dig.resolve(`${flippedIp}.${rblServer.dns}`, 'TXT', DNS_OPTIONS));
result.txtRecords = error2 || !txtRecords ? [] : txtRecords.map(x => x.join('')); result.txtRecords = error2 || !txtRecords ? [] : txtRecords.map(x => x.join(''));
debug(`checkRbl (${domain}) error: ${error2?.message || null} txtRecords: ${JSON.stringify(txtRecords)}`); log(`checkRbl (${domain}) error: ${error2?.message || null} txtRecords: ${JSON.stringify(txtRecords)}`);
blockedServers.push(result); blockedServers.push(result);
} }
@@ -573,11 +573,11 @@ async function getStatus(domain) {
for (let i = 0; i < checks.length; i++) { for (let i = 0; i < checks.length; i++) {
const response = responses[i], check = checks[i]; const response = responses[i], check = checks[i];
if (response.status !== 'fulfilled') { if (response.status !== 'fulfilled') {
debug(`check ${check.what} was rejected. This is not expected. reason: ${response.reason}`); log(`check ${check.what} was rejected. This is not expected. reason: ${response.reason}`);
continue; continue;
} }
if (response.value.message) debug(`${check.what} (${domain}): ${response.value.message}`); if (response.value.message) log(`${check.what} (${domain}): ${response.value.message}`);
safe.set(results, checks[i].what, response.value || {}); safe.set(results, checks[i].what, response.value || {});
} }
@@ -627,7 +627,7 @@ async function txtRecordsWithSpf(domain, mailFqdn) {
const txtRecords = await dns.getDnsRecords('', domain, 'TXT'); const txtRecords = await dns.getDnsRecords('', domain, 'TXT');
debug('txtRecordsWithSpf: current txt records - %j', txtRecords); log('txtRecordsWithSpf: current txt records - %j', txtRecords);
let i, matches, validSpf; let i, matches, validSpf;
@@ -644,10 +644,10 @@ async function txtRecordsWithSpf(domain, mailFqdn) {
if (!matches) { // no spf record was found, create one if (!matches) { // no spf record was found, create one
txtRecords.push('"v=spf1 a:' + mailFqdn + ' ~all"'); txtRecords.push('"v=spf1 a:' + mailFqdn + ' ~all"');
debug('txtRecordsWithSpf: adding txt record'); log('txtRecordsWithSpf: adding txt record');
} else { // just add ourself } else { // just add ourself
txtRecords[i] = matches[1] + ' a:' + mailFqdn + txtRecords[i].slice(matches[1].length); txtRecords[i] = matches[1] + ' a:' + mailFqdn + txtRecords[i].slice(matches[1].length);
debug('txtRecordsWithSpf: inserting txt record'); log('txtRecordsWithSpf: inserting txt record');
} }
return txtRecords; return txtRecords;
@@ -657,7 +657,7 @@ async function upsertDnsRecords(domain, mailFqdn) {
assert.strictEqual(typeof domain, 'string'); assert.strictEqual(typeof domain, 'string');
assert.strictEqual(typeof mailFqdn, 'string'); assert.strictEqual(typeof mailFqdn, 'string');
debug(`upsertDnsRecords: updating mail dns records domain:${domain} mailFqdn:${mailFqdn}`); log(`upsertDnsRecords: updating mail dns records domain:${domain} mailFqdn:${mailFqdn}`);
const mailDomain = await getDomain(domain); const mailDomain = await getDomain(domain);
if (!mailDomain) throw new BoxError(BoxError.NOT_FOUND, 'mail domain not found'); if (!mailDomain) throw new BoxError(BoxError.NOT_FOUND, 'mail domain not found');
@@ -679,13 +679,13 @@ async function upsertDnsRecords(domain, mailFqdn) {
const dmarcRecords = await dns.getDnsRecords('_dmarc', domain, 'TXT'); // only update dmarc if absent. this allows user to set email for reporting const dmarcRecords = await dns.getDnsRecords('_dmarc', domain, 'TXT'); // only update dmarc if absent. this allows user to set email for reporting
if (dmarcRecords.length === 0) records.push({ subdomain: '_dmarc', domain, type: 'TXT', values: [ '"v=DMARC1; p=reject; pct=100"' ] }); if (dmarcRecords.length === 0) records.push({ subdomain: '_dmarc', domain, type: 'TXT', values: [ '"v=DMARC1; p=reject; pct=100"' ] });
debug(`upsertDnsRecords: updating ${domain} with ${records.length} records: ${JSON.stringify(records)}`); log(`upsertDnsRecords: updating ${domain} with ${records.length} records: ${JSON.stringify(records)}`);
for (const record of records) { for (const record of records) {
await dns.upsertDnsRecords(record.subdomain, record.domain, record.type, record.values); await dns.upsertDnsRecords(record.subdomain, record.domain, record.type, record.values);
} }
debug(`upsertDnsRecords: records of ${domain} added`); log(`upsertDnsRecords: records of ${domain} added`);
} }
async function setDnsRecords(domain) { async function setDnsRecords(domain) {
@@ -714,7 +714,7 @@ async function setMailFromValidation(domain, enabled) {
await updateDomain(domain, { mailFromValidation: enabled }); await updateDomain(domain, { mailFromValidation: enabled });
safe(mailServer.restart(), { debug }); // have to restart mail container since haraka cannot watch symlinked config files (mail.ini) safe(mailServer.restart(), { debug: log }); // have to restart mail container since haraka cannot watch symlinked config files (mail.ini)
} }
async function setBanner(domain, banner) { async function setBanner(domain, banner) {
@@ -723,7 +723,7 @@ async function setBanner(domain, banner) {
await updateDomain(domain, { banner }); await updateDomain(domain, { banner });
safe(mailServer.restart(), { debug }); safe(mailServer.restart(), { debug: log });
} }
async function setCatchAllAddress(domain, addresses) { async function setCatchAllAddress(domain, addresses) {
@@ -736,7 +736,7 @@ async function setCatchAllAddress(domain, addresses) {
await updateDomain(domain, { catchAll: addresses }); await updateDomain(domain, { catchAll: addresses });
safe(mailServer.restart(), { debug }); // have to restart mail container since haraka cannot watch symlinked config files (mail.ini) safe(mailServer.restart(), { debug: log }); // have to restart mail container since haraka cannot watch symlinked config files (mail.ini)
} }
async function setMailRelay(domain, relay, options) { async function setMailRelay(domain, relay, options) {
@@ -760,7 +760,7 @@ async function setMailRelay(domain, relay, options) {
await updateDomain(domain, { relay }); await updateDomain(domain, { relay });
safe(mailServer.restart(), { debug }); safe(mailServer.restart(), { debug: log });
} }
async function setMailEnabled(domain, enabled, auditSource) { async function setMailEnabled(domain, enabled, auditSource) {
@@ -977,7 +977,7 @@ async function delMailbox(name, domain, options, auditSource) {
if (result.affectedRows === 0) throw new BoxError(BoxError.NOT_FOUND, 'Mailbox not found'); if (result.affectedRows === 0) throw new BoxError(BoxError.NOT_FOUND, 'Mailbox not found');
const [error] = await safe(removeSolrIndex(mailbox)); const [error] = await safe(removeSolrIndex(mailbox));
if (error) debug(`delMailbox: failed to remove solr index: ${error.message}`); if (error) log(`delMailbox: failed to remove solr index: ${error.message}`);
await eventlog.add(eventlog.ACTION_MAIL_MAILBOX_REMOVE, auditSource, { name, domain }); await eventlog.add(eventlog.ACTION_MAIL_MAILBOX_REMOVE, auditSource, { name, domain });
} }
@@ -1190,7 +1190,7 @@ async function resolveMailingList(listName, listDomain) {
const member =`${memberName}@${memberDomain}`; // cleaned up without any '+' subaddress const member =`${memberName}@${memberDomain}`; // cleaned up without any '+' subaddress
if (visited.includes(member)) { if (visited.includes(member)) {
debug(`resolveMailingList: list ${listName}@${listDomain} has a recursion at member ${member}`); log(`resolveMailingList: list ${listName}@${listDomain} has a recursion at member ${member}`);
continue; continue;
} }
visited.push(member); visited.push(member);

10
src/mailer.js
View File

@@ -3,7 +3,7 @@ import BoxError from './boxerror.js';
import branding from './branding.js'; import branding from './branding.js';
import constants from './constants.js'; import constants from './constants.js';
import dashboard from './dashboard.js'; import dashboard from './dashboard.js';
import debugModule from 'debug'; import logger from './logger.js';
import ejs from 'ejs'; import ejs from 'ejs';
import mailServer from './mailserver.js'; import mailServer from './mailserver.js';
import nodemailer from 'nodemailer'; import nodemailer from 'nodemailer';
@@ -11,7 +11,7 @@ import path from 'node:path';
import safe from 'safetydance'; import safe from 'safetydance';
import translations from './translations.js'; import translations from './translations.js';
const debug = debugModule('box:mailer'); const { log, trace } = logger('mailer');
const _mailQueue = []; // accumulate mails in test mode; const _mailQueue = []; // accumulate mails in test mode;
@@ -63,7 +63,7 @@ async function sendMail(mailOptions) {
const [error] = await safe(transport.sendMail(mailOptions)); const [error] = await safe(transport.sendMail(mailOptions));
if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, `Error sending Email "${mailOptions.subject}" to ${mailOptions.to}: ${error.message}`); if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, `Error sending Email "${mailOptions.subject}" to ${mailOptions.to}: ${error.message}`);
debug(`Email "${mailOptions.subject}" sent to ${mailOptions.to}`); log(`Email "${mailOptions.subject}" sent to ${mailOptions.to}`);
} }
function render(templateFile, params, translationAssets) { function render(templateFile, params, translationAssets) {
@@ -73,7 +73,7 @@ function render(templateFile, params, translationAssets) {
let content = null; let content = null;
let raw = safe.fs.readFileSync(path.join(MAIL_TEMPLATES_DIR, templateFile), 'utf8'); let raw = safe.fs.readFileSync(path.join(MAIL_TEMPLATES_DIR, templateFile), 'utf8');
if (raw === null) { if (raw === null) {
debug(`Error loading ${templateFile}`); log(`Error loading ${templateFile}`);
return ''; return '';
} }
@@ -82,7 +82,7 @@ function render(templateFile, params, translationAssets) {
try { try {
content = ejs.render(raw, params); content = ejs.render(raw, params);
} catch (e) { } catch (e) {
debug(`Error rendering ${templateFile}`, e); log(`Error rendering ${templateFile}`, e);
} }
return content; return content;

28
src/mailserver.js
View File

@@ -1,7 +1,7 @@
import assert from 'node:assert'; import assert from 'node:assert';
import BoxError from './boxerror.js'; import BoxError from './boxerror.js';
import constants from './constants.js'; import constants from './constants.js';
import debugModule from 'debug'; import logger from './logger.js';
import dns from './dns.js'; import dns from './dns.js';
import docker from './docker.js'; import docker from './docker.js';
import domains from './domains.js'; import domains from './domains.js';
@@ -22,7 +22,7 @@ import shellModule from './shell.js';
import tasks from './tasks.js'; import tasks from './tasks.js';
import users from './users.js'; import users from './users.js';
const debug = debugModule('box:mailserver'); const { log, trace } = logger('mailserver');
const shell = shellModule('mailserver'); const shell = shellModule('mailserver');
const DEFAULT_MEMORY_LIMIT = 512 * 1024 * 1024; const DEFAULT_MEMORY_LIMIT = 512 * 1024 * 1024;
@@ -31,7 +31,7 @@ const DEFAULT_MEMORY_LIMIT = 512 * 1024 * 1024;
async function createMailConfig(mailFqdn) { async function createMailConfig(mailFqdn) {
assert.strictEqual(typeof mailFqdn, 'string'); assert.strictEqual(typeof mailFqdn, 'string');
debug(`createMailConfig: generating mail config with ${mailFqdn}`); log(`createMailConfig: generating mail config with ${mailFqdn}`);
const mailDomains = await mail.listDomains(); const mailDomains = await mail.listDomains();
@@ -124,7 +124,7 @@ async function configureMail(mailFqdn, mailDomain, serviceConfig) {
// if the 'yellowtent' user of OS and the 'cloudron' user of mail container don't match, the keys become inaccessible by mail code // if the 'yellowtent' user of OS and the 'cloudron' user of mail container don't match, the keys become inaccessible by mail code
if (!safe.fs.chmodSync(mailKeyFilePath, 0o644)) throw new BoxError(BoxError.FS_ERROR, `Could not chmod key file: ${safe.error.message}`); if (!safe.fs.chmodSync(mailKeyFilePath, 0o644)) throw new BoxError(BoxError.FS_ERROR, `Could not chmod key file: ${safe.error.message}`);
debug('configureMail: stopping and deleting previous mail container'); log('configureMail: stopping and deleting previous mail container');
await docker.stopContainer('mail'); await docker.stopContainer('mail');
await docker.deleteContainer('mail'); await docker.deleteContainer('mail');
@@ -156,7 +156,7 @@ async function configureMail(mailFqdn, mailDomain, serviceConfig) {
--label isCloudronManaged=true \ --label isCloudronManaged=true \
${readOnly} -v /run -v /tmp ${image} ${cmd}`; ${readOnly} -v /run -v /tmp ${image} ${cmd}`;
debug('configureMail: starting mail container'); log('configureMail: starting mail container');
await shell.bash(runCmd, { encoding: 'utf8' }); await shell.bash(runCmd, { encoding: 'utf8' });
} }
@@ -172,7 +172,7 @@ async function restart() {
const mailConfig = await services.getServiceConfig('mail'); const mailConfig = await services.getServiceConfig('mail');
const { domain, fqdn } = await getLocation(); const { domain, fqdn } = await getLocation();
debug(`restart: restarting mail container with mailFqdn:${fqdn} mailDomain:${domain}`); log(`restart: restarting mail container with mailFqdn:${fqdn} mailDomain:${domain}`);
// NOTE: the email container has to be re-created. this is because some of the settings like solr config rely on starting with a clean /run state // NOTE: the email container has to be re-created. this is because some of the settings like solr config rely on starting with a clean /run state
await locks.wait(locks.TYPE_MAIL_SERVER_RESTART); await locks.wait(locks.TYPE_MAIL_SERVER_RESTART);
@@ -184,7 +184,7 @@ async function restart() {
async function start(existingInfra) { async function start(existingInfra) {
assert.strictEqual(typeof existingInfra, 'object'); assert.strictEqual(typeof existingInfra, 'object');
debug('startMail: starting'); log('startMail: starting');
await restart(); await restart();
if (existingInfra.version !== 'none' && existingInfra.images.mail !== infra.images.mail) await docker.deleteImage(existingInfra.images.mail); if (existingInfra.version !== 'none' && existingInfra.images.mail !== infra.images.mail) await docker.deleteImage(existingInfra.images.mail);
@@ -194,11 +194,11 @@ async function restartIfActivated() {
const activated = await users.isActivated(); const activated = await users.isActivated();
if (!activated) { if (!activated) {
debug('restartIfActivated: skipping restart of mail container since Cloudron is not activated yet'); log('restartIfActivated: skipping restart of mail container since Cloudron is not activated yet');
return; // not provisioned yet, do not restart container after dns setup return; // not provisioned yet, do not restart container after dns setup
} }
debug('restartIfActivated: restarting on activated'); log('restartIfActivated: restarting on activated');
await restart(); await restart();
} }
@@ -208,7 +208,7 @@ async function onDomainAdded(domain) {
const { fqdn } = await getLocation(); const { fqdn } = await getLocation();
if (!fqdn) return; // mail domain is not set yet (when provisioning) if (!fqdn) return; // mail domain is not set yet (when provisioning)
debug(`onDomainAdded: configuring mail for added domain ${domain}`); log(`onDomainAdded: configuring mail for added domain ${domain}`);
await mail.upsertDnsRecords(domain, fqdn); await mail.upsertDnsRecords(domain, fqdn);
await restartIfActivated(); await restartIfActivated();
} }
@@ -216,7 +216,7 @@ async function onDomainAdded(domain) {
async function onDomainRemoved(domain) { async function onDomainRemoved(domain) {
assert.strictEqual(typeof domain, 'string'); assert.strictEqual(typeof domain, 'string');
debug(`onDomainRemoved: configuring mail for removed domain ${domain}`); log(`onDomainRemoved: configuring mail for removed domain ${domain}`);
await restart(); await restart();
} }
@@ -224,10 +224,10 @@ async function checkCertificate() {
const certificate = await reverseProxy.getMailCertificate(); const certificate = await reverseProxy.getMailCertificate();
const cert = safe.fs.readFileSync(`${paths.MAIL_CONFIG_DIR}/tls_cert.pem`, { encoding: 'utf8' }); const cert = safe.fs.readFileSync(`${paths.MAIL_CONFIG_DIR}/tls_cert.pem`, { encoding: 'utf8' });
if (cert === certificate.cert) { if (cert === certificate.cert) {
debug('checkCertificate: certificate has not changed'); log('checkCertificate: certificate has not changed');
return; return;
} }
debug('checkCertificate: certificate has changed'); log('checkCertificate: certificate has changed');
await restartIfActivated(); await restartIfActivated();
} }
@@ -289,7 +289,7 @@ async function startChangeLocation(subdomain, domain, auditSource) {
.then(async () => { .then(async () => {
await platform.onMailServerLocationChanged(auditSource); await platform.onMailServerLocationChanged(auditSource);
}) })
.catch((taskError) => debug(`startChangeLocation`, taskError)); .catch((taskError) => log(`startChangeLocation`, taskError));
await eventlog.add(eventlog.ACTION_MAIL_LOCATION, auditSource, { subdomain, domain, taskId }); await eventlog.add(eventlog.ACTION_MAIL_LOCATION, auditSource, { subdomain, domain, taskId });
return taskId; return taskId;

12
src/metrics.js
View File

@@ -2,7 +2,7 @@ import apps from './apps.js';
import assert from 'node:assert'; import assert from 'node:assert';
import BoxError from './boxerror.js'; import BoxError from './boxerror.js';
import constants from './constants.js'; import constants from './constants.js';
import debugModule from 'debug'; import logger from './logger.js';
import docker from './docker.js'; import docker from './docker.js';
import fs from 'node:fs'; import fs from 'node:fs';
import net from 'node:net'; import net from 'node:net';
@@ -15,7 +15,7 @@ import shellModule from './shell.js';
import superagent from '@cloudron/superagent'; import superagent from '@cloudron/superagent';
import _ from './underscore.js'; import _ from './underscore.js';
const debug = debugModule('box:metrics'); const { log, trace } = logger('metrics');
const shell = shellModule('metrics'); const shell = shellModule('metrics');
@@ -163,7 +163,7 @@ async function readSystemMetrics() {
} }
async function sendToGraphite() { async function sendToGraphite() {
// debug('sendStatsToGraphite: collecting stats'); // log('sendStatsToGraphite: collecting stats');
const result = await readSystemMetrics(); const result = await readSystemMetrics();
@@ -203,7 +203,7 @@ async function sendToGraphite() {
}); });
client.on('error', (error) => { client.on('error', (error) => {
debug(`Error sending data to graphite: ${error.message}`); log(`Error sending data to graphite: ${error.message}`);
resolve(); resolve();
}); });
@@ -381,7 +381,7 @@ async function pipeContainerToMap(name, statsMap) {
// we used to poll before instead of a stream. but docker caches metrics internally and rate logic has to handle dups // we used to poll before instead of a stream. but docker caches metrics internally and rate logic has to handle dups
const statsStream = await docker.getStats(name, { stream: true }); const statsStream = await docker.getStats(name, { stream: true });
statsStream.on('error', (error) => debug(error)); statsStream.on('error', (error) => log(error));
statsStream.on('data', (data) => { statsStream.on('data', (data) => {
const stats = JSON.parse(data.toString('utf8')); const stats = JSON.parse(data.toString('utf8'));
const metrics = translateContainerStatsSync(stats); const metrics = translateContainerStatsSync(stats);
@@ -478,7 +478,7 @@ async function getStream(options) {
const INTERVAL_MSECS = 1000; const INTERVAL_MSECS = 1000;
intervalId = setInterval(async () => { intervalId = setInterval(async () => {
if (options.system) await safe(pipeSystemToMap(statsMap), { debug }); if (options.system) await safe(pipeSystemToMap(statsMap), { debug: log });
const result = {}; const result = {};
const nowSecs = Date.now() / 1000; // to match graphite return value const nowSecs = Date.now() / 1000; // to match graphite return value

6
src/mounts.js
View File

@@ -1,7 +1,7 @@
import assert from 'node:assert'; import assert from 'node:assert';
import BoxError from './boxerror.js'; import BoxError from './boxerror.js';
import constants from './constants.js'; import constants from './constants.js';
import debugModule from 'debug'; import logger from './logger.js';
import ejs from 'ejs'; import ejs from 'ejs';
import fs from 'node:fs'; import fs from 'node:fs';
import path from 'node:path'; import path from 'node:path';
@@ -9,7 +9,7 @@ import paths from './paths.js';
import safe from 'safetydance'; import safe from 'safetydance';
import shellModule from './shell.js'; import shellModule from './shell.js';
const debug = debugModule('box:mounts'); const { log, trace } = logger('mounts');
const shell = shellModule('mounts'); const shell = shellModule('mounts');
const MOUNT_TYPE_FILESYSTEM = 'filesystem'; const MOUNT_TYPE_FILESYSTEM = 'filesystem';
@@ -163,7 +163,7 @@ async function removeMount(mount) {
if (constants.TEST) return; if (constants.TEST) return;
await safe(shell.sudo([ RM_MOUNT_CMD, hostPath ], {}), { debug }); // ignore any error await safe(shell.sudo([ RM_MOUNT_CMD, hostPath ], {}), { debug: log }); // ignore any error
if (mountType === MOUNT_TYPE_SSHFS) { if (mountType === MOUNT_TYPE_SSHFS) {
const keyFilePath = path.join(paths.SSHFS_KEYS_DIR, `identity_file_${path.basename(hostPath)}`); const keyFilePath = path.join(paths.SSHFS_KEYS_DIR, `identity_file_${path.basename(hostPath)}`);

10
src/network/generic.js
View File

@@ -1,11 +1,11 @@
import assert from 'node:assert'; import assert from 'node:assert';
import BoxError from '../boxerror.js'; import BoxError from '../boxerror.js';
import constants from '../constants.js'; import constants from '../constants.js';
import debugModule from 'debug'; import logger from '../logger.js';
import safe from 'safetydance'; import safe from 'safetydance';
import superagent from '@cloudron/superagent'; import superagent from '@cloudron/superagent';
const debug = debugModule('box:network/generic'); const { log, trace } = logger('network/generic');
const gCache = { ipv4: {}, ipv6: {} }; // each has { timestamp, value, request } const gCache = { ipv4: {}, ipv6: {} }; // each has { timestamp, value, request }
@@ -15,16 +15,16 @@ async function getIP(type) {
gCache[type].value = null; // clear the obsolete value gCache[type].value = null; // clear the obsolete value
debug(`getIP: querying ${url} to get ${type}`); log(`getIP: querying ${url} to get ${type}`);
const [networkError, response] = await safe(superagent.get(url).timeout(30 * 1000).retry(2).ok(() => true)); const [networkError, response] = await safe(superagent.get(url).timeout(30 * 1000).retry(2).ok(() => true));
if (networkError || response.status !== 200) { if (networkError || response.status !== 200) {
debug(`getIP: Error getting IP. ${networkError.message}`); log(`getIP: Error getting IP. ${networkError.message}`);
throw new BoxError(BoxError.EXTERNAL_ERROR, `Unable to detect ${type}. API server (${type}.api.cloudron.io) unreachable`); throw new BoxError(BoxError.EXTERNAL_ERROR, `Unable to detect ${type}. API server (${type}.api.cloudron.io) unreachable`);
} }
if (!response.body?.ip) { if (!response.body?.ip) {
debug('get: Unexpected answer. No "ip" found in response body.', response.body); log('get: Unexpected answer. No "ip" found in response body.', response.body);
throw new BoxError(BoxError.EXTERNAL_ERROR, `Unable to detect ${type}. No IP found in response`); throw new BoxError(BoxError.EXTERNAL_ERROR, `Unable to detect ${type}. No IP found in response`);
} }

8
src/network/network-interface.js
View File

@@ -1,10 +1,10 @@
import assert from 'node:assert'; import assert from 'node:assert';
import BoxError from '../boxerror.js'; import BoxError from '../boxerror.js';
import debugModule from 'debug'; import logger from '../logger.js';
import os from 'node:os'; import os from 'node:os';
import safe from 'safetydance'; import safe from 'safetydance';
const debug = debugModule('box:network/network-interface'); const { log, trace } = logger('network/network-interface');
async function getIPv4(config) { async function getIPv4(config) {
@@ -16,7 +16,7 @@ async function getIPv4(config) {
const addresses = iface.filter(i => i.family === 'IPv4').map(i => i.address); const addresses = iface.filter(i => i.family === 'IPv4').map(i => i.address);
if (addresses.length === 0) throw new BoxError(BoxError.NETWORK_ERROR, `${config.ifname} does not have any IPv4 address`); if (addresses.length === 0) throw new BoxError(BoxError.NETWORK_ERROR, `${config.ifname} does not have any IPv4 address`);
if (addresses.length > 1) debug(`${config.ifname} has multiple ipv4 - ${JSON.stringify(addresses)}. choosing the first one.`); if (addresses.length > 1) log(`${config.ifname} has multiple ipv4 - ${JSON.stringify(addresses)}. choosing the first one.`);
return addresses[0]; return addresses[0];
} }
@@ -30,7 +30,7 @@ async function getIPv6(config) {
const addresses = iface.filter(i => i.family === 'IPv6').map(i => i.address); const addresses = iface.filter(i => i.family === 'IPv6').map(i => i.address);
if (addresses.length === 0) throw new BoxError(BoxError.NETWORK_ERROR, `${config.ifname} does not have any IPv6 address`); if (addresses.length === 0) throw new BoxError(BoxError.NETWORK_ERROR, `${config.ifname} does not have any IPv6 address`);
if (addresses.length > 1) debug(`${config.ifname} has multiple ipv6 - ${JSON.stringify(addresses)}. choosing the first one.`); if (addresses.length > 1) log(`${config.ifname} has multiple ipv6 - ${JSON.stringify(addresses)}. choosing the first one.`);
return addresses[0]; return addresses[0];
} }

22
src/notifications.js
View File

@@ -4,13 +4,13 @@ import BoxError from './boxerror.js';
import changelog from './changelog.js'; import changelog from './changelog.js';
import dashboard from './dashboard.js'; import dashboard from './dashboard.js';
import database from './database.js'; import database from './database.js';
import debugModule from 'debug'; import logger from './logger.js';
import eventlog from './eventlog.js'; import eventlog from './eventlog.js';
import mailer from './mailer.js'; import mailer from './mailer.js';
import safe from 'safetydance'; import safe from 'safetydance';
import users from './users.js'; import users from './users.js';
const debug = debugModule('box:notifications'); const { log, trace } = logger('notifications');
const TYPE_CLOUDRON_INSTALLED = 'cloudronInstalled'; const TYPE_CLOUDRON_INSTALLED = 'cloudronInstalled';
const TYPE_CLOUDRON_UPDATED = 'cloudronUpdated'; const TYPE_CLOUDRON_UPDATED = 'cloudronUpdated';
@@ -44,7 +44,7 @@ async function add(type, title, message, data) {
assert.strictEqual(typeof message, 'string'); assert.strictEqual(typeof message, 'string');
assert.strictEqual(typeof data, 'object'); assert.strictEqual(typeof data, 'object');
debug(`add: ${type} ${title}`); log(`add: ${type} ${title}`);
const query = 'INSERT INTO notifications (type, title, message, acknowledged, eventId, context) VALUES (?, ?, ?, ?, ?, ?)'; const query = 'INSERT INTO notifications (type, title, message, acknowledged, eventId, context) VALUES (?, ?, ?, ?, ?, ?)';
const args = [ type, title, message, false, data?.eventId || null, data.context || '' ]; const args = [ type, title, message, false, data?.eventId || null, data.context || '' ];
@@ -147,7 +147,7 @@ async function oomEvent(eventId, containerId, app, addonName, event) {
const admins = await users.getAdmins(); const admins = await users.getAdmins();
for (const admin of admins) { for (const admin of admins) {
if (admin.notificationConfig.includes(TYPE_APP_OOM)) { if (admin.notificationConfig.includes(TYPE_APP_OOM)) {
await safe(mailer.oomEvent(admin.email, containerId, app, addonName, event), { debug }); await safe(mailer.oomEvent(admin.email, containerId, app, addonName, event), { debug: log });
} }
} }
} }
@@ -159,7 +159,7 @@ async function appUp(eventId, app) {
const admins = await users.getAdmins(); const admins = await users.getAdmins();
for (const admin of admins) { for (const admin of admins) {
if (admin.notificationConfig.includes(TYPE_APP_UP)) { if (admin.notificationConfig.includes(TYPE_APP_UP)) {
await safe(mailer.appUp(admin.email, app), { debug }); await safe(mailer.appUp(admin.email, app), { debug: log });
} }
} }
} }
@@ -171,7 +171,7 @@ async function appDown(eventId, app) {
const admins = await users.getAdmins(); const admins = await users.getAdmins();
for (const admin of admins) { for (const admin of admins) {
if (admin.notificationConfig.includes(TYPE_APP_DOWN)) { if (admin.notificationConfig.includes(TYPE_APP_DOWN)) {
await safe(mailer.appDown(admin.email, app), { debug }); await safe(mailer.appDown(admin.email, app), { debug: log });
} }
} }
} }
@@ -222,7 +222,7 @@ async function boxUpdateError(eventId, errorMessage) {
const admins = await users.getAdmins(); const admins = await users.getAdmins();
for (const admin of admins) { for (const admin of admins) {
if (admin.notificationConfig.includes(TYPE_CLOUDRON_UPDATE_FAILED)) { if (admin.notificationConfig.includes(TYPE_CLOUDRON_UPDATE_FAILED)) {
await safe(mailer.boxUpdateError(admin.email, errorMessage), { debug }); await safe(mailer.boxUpdateError(admin.email, errorMessage), { debug: log });
} }
} }
} }
@@ -237,7 +237,7 @@ async function certificateRenewalError(eventId, fqdn, errorMessage) {
const admins = await users.getAdmins(); const admins = await users.getAdmins();
for (const admin of admins) { for (const admin of admins) {
if (admin.notificationConfig.includes(TYPE_CERTIFICATE_RENEWAL_FAILED)) { if (admin.notificationConfig.includes(TYPE_CERTIFICATE_RENEWAL_FAILED)) {
await safe(mailer.certificateRenewalError(admin.email, fqdn, errorMessage), { debug }); await safe(mailer.certificateRenewalError(admin.email, fqdn, errorMessage), { debug: log });
} }
} }
} }
@@ -253,7 +253,7 @@ async function backupFailed(eventId, taskId, errorMessage) {
const superadmins = await users.getSuperadmins(); const superadmins = await users.getSuperadmins();
for (const superadmin of superadmins) { for (const superadmin of superadmins) {
if (superadmin.notificationConfig.includes(TYPE_BACKUP_FAILED)) { if (superadmin.notificationConfig.includes(TYPE_BACKUP_FAILED)) {
await safe(mailer.backupFailed(superadmin.email, errorMessage, `https://${dashboardFqdn}/logs.html?taskId=${taskId}`), { debug }); await safe(mailer.backupFailed(superadmin.email, errorMessage, `https://${dashboardFqdn}/logs.html?taskId=${taskId}`), { debug: log });
} }
} }
} }
@@ -262,7 +262,7 @@ async function rebootRequired() {
const admins = await users.getAdmins(); const admins = await users.getAdmins();
for (const admin of admins) { for (const admin of admins) {
if (admin.notificationConfig.includes(TYPE_REBOOT)) { if (admin.notificationConfig.includes(TYPE_REBOOT)) {
await safe(mailer.rebootRequired(admin.email), { debug }); await safe(mailer.rebootRequired(admin.email), { debug: log });
} }
} }
} }
@@ -273,7 +273,7 @@ async function lowDiskSpace(message) {
const admins = await users.getAdmins(); const admins = await users.getAdmins();
for (const admin of admins) { for (const admin of admins) {
if (admin.notificationConfig.includes(TYPE_DISK_SPACE)) { if (admin.notificationConfig.includes(TYPE_DISK_SPACE)) {
await safe(mailer.lowDiskSpace(admin.email, message), { debug }); await safe(mailer.lowDiskSpace(admin.email, message), { debug: log });
} }
} }
} }

52
src/oidcserver.js
View File

@@ -7,7 +7,7 @@ import branding from './branding.js';
import constants from './constants.js'; import constants from './constants.js';
import crypto from 'node:crypto'; import crypto from 'node:crypto';
import dashboard from './dashboard.js'; import dashboard from './dashboard.js';
import debugModule from 'debug'; import logger from './logger.js';
import dns from './dns.js'; import dns from './dns.js';
import ejs from 'ejs'; import ejs from 'ejs';
import express from 'express'; import express from 'express';
@@ -33,7 +33,7 @@ import util from 'node:util';
import Provider from 'oidc-provider'; import Provider from 'oidc-provider';
import mailpasswords from './mailpasswords.js'; import mailpasswords from './mailpasswords.js';
const debug = debugModule('box:oidcserver'); const { log, trace } = logger('oidcserver');
// 1. Index.vue starts the OIDC flow by navigating to /openid/auth. Webadmin sets callback url to authcallback.html + implicit flow // 1. Index.vue starts the OIDC flow by navigating to /openid/auth. Webadmin sets callback url to authcallback.html + implicit flow
@@ -83,12 +83,12 @@ class StorageAdapter {
} }
constructor(name) { constructor(name) {
debug(`Creating OpenID storage adapter for ${name}`); log(`Creating OpenID storage adapter for ${name}`);
this.name = name; this.name = name;
} }
async upsert(id, payload, expiresIn) { async upsert(id, payload, expiresIn) {
debug(`[${this.name}] upsert: ${id}`); log(`[${this.name}] upsert: ${id}`);
const expiresAt = expiresIn ? new Date(Date.now() + (expiresIn * 1000)) : 0; const expiresAt = expiresIn ? new Date(Date.now() + (expiresIn * 1000)) : 0;
@@ -102,7 +102,7 @@ class StorageAdapter {
const [error] = await safe(tokens.add({ clientId: payload.clientId, identifier: user.id, expires, accessToken: id, allowedIpRanges: '' })); const [error] = await safe(tokens.add({ clientId: payload.clientId, identifier: user.id, expires, accessToken: id, allowedIpRanges: '' }));
if (error) { if (error) {
debug('Error adding access token', error); log('Error adding access token', error);
throw error; throw error;
} }
} else { } else {
@@ -111,12 +111,12 @@ class StorageAdapter {
} }
async find(id) { async find(id) {
debug(`[${this.name}] find: ${id}`); log(`[${this.name}] find: ${id}`);
if (this.name === 'Client') { if (this.name === 'Client') {
const [error, client] = await safe(oidcClients.get(id)); const [error, client] = await safe(oidcClients.get(id));
if (error || !client) { if (error || !client) {
debug('find: error getting client', error); log('find: error getting client', error);
return null; return null;
} }
@@ -132,7 +132,7 @@ class StorageAdapter {
if (client.appId) { if (client.appId) {
const [appError, app] = await safe(apps.get(client.appId)); const [appError, app] = await safe(apps.get(client.appId));
if (appError || !app) { if (appError || !app) {
debug(`find: Unknown app for client with appId ${client.appId}`); log(`find: Unknown app for client with appId ${client.appId}`);
return null; return null;
} }
@@ -183,12 +183,12 @@ class StorageAdapter {
} }
async findByUserCode(userCode) { async findByUserCode(userCode) {
debug(`[${this.name}] FIXME findByUserCode userCode:${userCode}`); log(`[${this.name}] FIXME findByUserCode userCode:${userCode}`);
} }
// this is called only on Session store. there is a payload.uid // this is called only on Session store. there is a payload.uid
async findByUid(uid) { async findByUid(uid) {
debug(`[${this.name}] findByUid: ${uid}`); log(`[${this.name}] findByUid: ${uid}`);
const data = await StorageAdapter.getData(this.name); const data = await StorageAdapter.getData(this.name);
for (const d in data) { for (const d in data) {
@@ -199,19 +199,19 @@ class StorageAdapter {
} }
async consume(id) { async consume(id) {
debug(`[${this.name}] consume: ${id}`); log(`[${this.name}] consume: ${id}`);
await StorageAdapter.updateData(this.name, (data) => data[id].consumed = true); await StorageAdapter.updateData(this.name, (data) => data[id].consumed = true);
} }
async destroy(id) { async destroy(id) {
debug(`[${this.name}] destroy: ${id}`); log(`[${this.name}] destroy: ${id}`);
await StorageAdapter.updateData(this.name, (data) => delete data[id]); await StorageAdapter.updateData(this.name, (data) => delete data[id]);
} }
async revokeByGrantId(grantId) { async revokeByGrantId(grantId) {
debug(`[${this.name}] revokeByGrantId: ${grantId}`); log(`[${this.name}] revokeByGrantId: ${grantId}`);
await StorageAdapter.updateData(this.name, (data) => { await StorageAdapter.updateData(this.name, (data) => {
for (const d in data) { for (const d in data) {
@@ -256,7 +256,7 @@ async function consumeAuthCode(authCode) {
// This exposed to run on a cron job // This exposed to run on a cron job
async function cleanupExpired() { async function cleanupExpired() {
debug('cleanupExpired'); log('cleanupExpired');
const types = [ 'AuthorizationCode', 'AccessToken', 'Grant', 'Interaction', 'RefreshToken', 'Session' ]; const types = [ 'AuthorizationCode', 'AccessToken', 'Grant', 'Interaction', 'RefreshToken', 'Session' ];
for (const type of types) { for (const type of types) {
@@ -282,7 +282,7 @@ async function renderError(error) {
language: await settings.get(settings.LANGUAGE_KEY), language: await settings.get(settings.LANGUAGE_KEY),
}; };
debug('renderError: %o', error); log('renderError: %o', error);
return ejs.render(TEMPLATE_ERROR, data); return ejs.render(TEMPLATE_ERROR, data);
} }
@@ -351,7 +351,7 @@ async function interactionLogin(req, res, next) {
const ip = req.headers['x-forwarded-for'] || req.socket.remoteAddress || null; const ip = req.headers['x-forwarded-for'] || req.socket.remoteAddress || null;
const clientId = details.params.client_id; const clientId = details.params.client_id;
debug(`interactionLogin: for OpenID client ${clientId} from ${ip}`); log(`interactionLogin: for OpenID client ${clientId} from ${ip}`);
if (req.body.autoLoginToken) { // auto login for first admin/owner if (req.body.autoLoginToken) { // auto login for first admin/owner
if (typeof req.body.autoLoginToken !== 'string') return next(new HttpError(400, 'autoLoginToken must be string if provided')); if (typeof req.body.autoLoginToken !== 'string') return next(new HttpError(400, 'autoLoginToken must be string if provided'));
@@ -394,10 +394,10 @@ async function interactionLogin(req, res, next) {
if (userPasskeys.length > 0) { if (userPasskeys.length > 0) {
const [passkeyError] = await safe(passkeys.verifyAuthentication(user, passkeyResponse)); const [passkeyError] = await safe(passkeys.verifyAuthentication(user, passkeyResponse));
if (passkeyError) { if (passkeyError) {
debug(`interactionLogin: passkey verification failed for ${username}: ${passkeyError.message}`); log(`interactionLogin: passkey verification failed for ${username}: ${passkeyError.message}`);
return next(new HttpError(401, 'Invalid passkey')); return next(new HttpError(401, 'Invalid passkey'));
} }
debug(`interactionLogin: passkey verified for ${username}`); log(`interactionLogin: passkey verified for ${username}`);
} }
} }
@@ -446,7 +446,7 @@ async function interactionConfirm(req, res, next) {
if (detailsError) return next(new HttpError(detailsError.statusCode, detailsError.error_description)); if (detailsError) return next(new HttpError(detailsError.statusCode, detailsError.error_description));
const { grantId, uid, prompt: { name, details }, params, session: { accountId }, lastSubmission } = interactionDetails; const { grantId, uid, prompt: { name, details }, params, session: { accountId }, lastSubmission } = interactionDetails;
debug(`route interaction confirm post uid:${uid} prompt.name:${name} accountId:${accountId}`); log(`route interaction confirm post uid:${uid} prompt.name:${name} accountId:${accountId}`);
const client = await oidcClients.get(params.client_id); const client = await oidcClients.get(params.client_id);
if (!client) return next(new Error('Client not found')); if (!client) return next(new Error('Client not found'));
@@ -510,7 +510,7 @@ async function interactionConfirm(req, res, next) {
const auditSource = AuditSource.fromOidcRequest(req); const auditSource = AuditSource.fromOidcRequest(req);
await eventlog.add(user.ghost ? eventlog.ACTION_USER_LOGIN_GHOST : eventlog.ACTION_USER_LOGIN, auditSource, { userId: user.id, user: users.removePrivateFields(user), appId: client.appId || null }); await eventlog.add(user.ghost ? eventlog.ACTION_USER_LOGIN_GHOST : eventlog.ACTION_USER_LOGIN, auditSource, { userId: user.id, user: users.removePrivateFields(user), appId: client.appId || null });
await safe(users.notifyLoginLocation(user, ip, userAgent, auditSource), { debug }); await safe(users.notifyLoginLocation(user, ip, userAgent, auditSource), { debug: log });
const result = { consent }; const result = { consent };
await gOidcProvider.interactionFinished(req, res, result, { mergeWithLastSubmission: true }); await gOidcProvider.interactionFinished(req, res, result, { mergeWithLastSubmission: true });
@@ -586,31 +586,31 @@ async function start() {
let keyEdDsa = await blobs.getString(blobs.OIDC_KEY_EDDSA); let keyEdDsa = await blobs.getString(blobs.OIDC_KEY_EDDSA);
if (!keyEdDsa) { if (!keyEdDsa) {
debug('Generating new OIDC EdDSA key'); log('Generating new OIDC EdDSA key');
const { privateKey } = await jose.generateKeyPair('EdDSA', { extractable: true }); const { privateKey } = await jose.generateKeyPair('EdDSA', { extractable: true });
keyEdDsa = Object.assign(await jose.exportJWK(privateKey), { alg: 'EdDSA' }); // alg is optional, but wp requires it keyEdDsa = Object.assign(await jose.exportJWK(privateKey), { alg: 'EdDSA' }); // alg is optional, but wp requires it
await blobs.setString(blobs.OIDC_KEY_EDDSA, JSON.stringify(keyEdDsa)); await blobs.setString(blobs.OIDC_KEY_EDDSA, JSON.stringify(keyEdDsa));
jwksKeys.push(keyEdDsa); jwksKeys.push(keyEdDsa);
} else { } else {
debug('Using existing OIDC EdDSA key'); log('Using existing OIDC EdDSA key');
jwksKeys.push(JSON.parse(keyEdDsa)); jwksKeys.push(JSON.parse(keyEdDsa));
} }
let keyRs256 = await blobs.getString(blobs.OIDC_KEY_RS256); let keyRs256 = await blobs.getString(blobs.OIDC_KEY_RS256);
if (!keyRs256) { if (!keyRs256) {
debug('Generating new OIDC RS256 key'); log('Generating new OIDC RS256 key');
const { privateKey } = await jose.generateKeyPair('RS256', { extractable: true }); const { privateKey } = await jose.generateKeyPair('RS256', { extractable: true });
keyRs256 = Object.assign(await jose.exportJWK(privateKey), { alg: 'RS256' }); // alg is optional, but wp requires it keyRs256 = Object.assign(await jose.exportJWK(privateKey), { alg: 'RS256' }); // alg is optional, but wp requires it
await blobs.setString(blobs.OIDC_KEY_RS256, JSON.stringify(keyRs256)); await blobs.setString(blobs.OIDC_KEY_RS256, JSON.stringify(keyRs256));
jwksKeys.push(keyRs256); jwksKeys.push(keyRs256);
} else { } else {
debug('Using existing OIDC RS256 key'); log('Using existing OIDC RS256 key');
jwksKeys.push(JSON.parse(keyRs256)); jwksKeys.push(JSON.parse(keyRs256));
} }
let cookieSecret = await settings.get(settings.OIDC_COOKIE_SECRET_KEY); let cookieSecret = await settings.get(settings.OIDC_COOKIE_SECRET_KEY);
if (!cookieSecret) { if (!cookieSecret) {
debug('Generating new cookie secret'); log('Generating new cookie secret');
cookieSecret = crypto.randomBytes(256).toString('base64'); cookieSecret = crypto.randomBytes(256).toString('base64');
await settings.set(settings.OIDC_COOKIE_SECRET_KEY, cookieSecret); await settings.set(settings.OIDC_COOKIE_SECRET_KEY, cookieSecret);
} }
@@ -725,7 +725,7 @@ async function start() {
const { subdomain, domain } = await dashboard.getLocation(); const { subdomain, domain } = await dashboard.getLocation();
const fqdn = dns.fqdn(subdomain, domain); const fqdn = dns.fqdn(subdomain, domain);
debug(`start: create provider for ${fqdn} at ${ROUTE_PREFIX}`); log(`start: create provider for ${fqdn} at ${ROUTE_PREFIX}`);
gOidcProvider = new Provider(`https://${fqdn}${ROUTE_PREFIX}`, configuration); gOidcProvider = new Provider(`https://${fqdn}${ROUTE_PREFIX}`, configuration);

6
src/once.js
View File

@@ -1,12 +1,12 @@
import debugModule from 'debug'; import logger from './logger.js';
const debug = debugModule('box:once'); const { log, trace } = logger('once');
// https://github.com/isaacs/once/blob/main/LICENSE (ISC) // https://github.com/isaacs/once/blob/main/LICENSE (ISC)
function once (fn) { function once (fn) {
const f = function () { const f = function () {
if (f.called) { if (f.called) {
debug(`${f.name} was already called, returning previous return value`); log(`${f.name} was already called, returning previous return value`);
return f.value; return f.value;
} }
f.called = true; f.called = true;

14
src/openssl.js
View File

@@ -1,19 +1,19 @@
import assert from 'node:assert'; import assert from 'node:assert';
import BoxError from './boxerror.js'; import BoxError from './boxerror.js';
import crypto from 'node:crypto'; import crypto from 'node:crypto';
import debugModule from 'debug'; import logger from './logger.js';
import fs from 'node:fs'; import fs from 'node:fs';
import os from 'node:os'; import os from 'node:os';
import path from 'node:path'; import path from 'node:path';
import safe from 'safetydance'; import safe from 'safetydance';
import shellModule from './shell.js'; import shellModule from './shell.js';
const debug = debugModule('box:openssl'); const { log, trace } = logger('openssl');
const shell = shellModule('openssl'); const shell = shellModule('openssl');
async function generateKey(type) { async function generateKey(type) {
debug(`generateKey: generating new key for${type}`); log(`generateKey: generating new key for${type}`);
if (type === 'rsa4096') { if (type === 'rsa4096') {
return await shell.spawn('openssl', ['genrsa', '4096'], { encoding: 'utf8' }); return await shell.spawn('openssl', ['genrsa', '4096'], { encoding: 'utf8' });
@@ -63,7 +63,7 @@ async function createCsr(key, cn, altNames) {
// while we pass the CN anyways, subjectAltName takes precedence // while we pass the CN anyways, subjectAltName takes precedence
const csrPem = await shell.spawn('openssl', ['req', '-new', '-key', keyFilePath, '-outform', 'PEM', '-subj', `/CN=${cn}`, '-config', opensslConfigFile], { encoding: 'utf8' }); const csrPem = await shell.spawn('openssl', ['req', '-new', '-key', keyFilePath, '-outform', 'PEM', '-subj', `/CN=${cn}`, '-config', opensslConfigFile], { encoding: 'utf8' });
await safe(fs.promises.rm(tmpdir, { recursive: true, force: true })); await safe(fs.promises.rm(tmpdir, { recursive: true, force: true }));
debug(`createCsr: csr file created for ${cn}`); log(`createCsr: csr file created for ${cn}`);
return csrPem; // inspect with openssl req -text -noout -in hostname.csr -inform pem return csrPem; // inspect with openssl req -text -noout -in hostname.csr -inform pem
}; };
@@ -81,7 +81,7 @@ async function getCertificateDates(cert) {
const notAfterDate = new Date(notAfter); const notAfterDate = new Date(notAfter);
const daysLeft = (notAfterDate - new Date())/(24 * 60 * 60 * 1000); const daysLeft = (notAfterDate - new Date())/(24 * 60 * 60 * 1000);
debug(`expiryDate: ${lines[2]} notBefore=${notBefore} notAfter=${notAfter} daysLeft=${daysLeft}`); log(`expiryDate: ${lines[2]} notBefore=${notBefore} notAfter=${notAfter} daysLeft=${daysLeft}`);
return { startDate: notBeforeDate, endDate: notAfterDate }; return { startDate: notBeforeDate, endDate: notAfterDate };
} }
@@ -106,7 +106,7 @@ async function generateCertificate(domain) {
const opensslConf = safe.fs.readFileSync('/etc/ssl/openssl.cnf', 'utf8'); const opensslConf = safe.fs.readFileSync('/etc/ssl/openssl.cnf', 'utf8');
const cn = domain; const cn = domain;
debug(`generateCertificate: domain=${domain} cn=${cn}`); log(`generateCertificate: domain=${domain} cn=${cn}`);
// SAN must contain all the domains since CN check is based on implementation if SAN is found. -checkhost also checks only SAN if present! // SAN must contain all the domains since CN check is based on implementation if SAN is found. -checkhost also checks only SAN if present!
const opensslConfWithSan = `${opensslConf}\n[SAN]\nsubjectAltName=DNS:${domain},DNS:*.${cn}\n`; const opensslConfWithSan = `${opensslConf}\n[SAN]\nsubjectAltName=DNS:${domain},DNS:*.${cn}\n`;
@@ -169,7 +169,7 @@ async function generateDkimKey() {
} }
async function generateDhparam() { async function generateDhparam() {
debug('generateDhparam: generating dhparams'); log('generateDhparam: generating dhparams');
return await shell.spawn('openssl', ['dhparam', '-dsaparam', '2048'], { encoding: 'utf8' }); return await shell.spawn('openssl', ['dhparam', '-dsaparam', '2048'], { encoding: 'utf8' });
} }

20
src/passkeys.js
View File

@@ -3,7 +3,7 @@ import BoxError from './boxerror.js';
import crypto from 'node:crypto'; import crypto from 'node:crypto';
import dashboard from './dashboard.js'; import dashboard from './dashboard.js';
import database from './database.js'; import database from './database.js';
import debugModule from 'debug'; import logger from './logger.js';
import safe from 'safetydance'; import safe from 'safetydance';
import { import {
generateRegistrationOptions, generateRegistrationOptions,
@@ -13,7 +13,7 @@ import {
} from '@simplewebauthn/server'; } from '@simplewebauthn/server';
import _ from './underscore.js'; import _ from './underscore.js';
const debug = debugModule('box:passkeys'); const { log, trace } = logger('passkeys');
const PASSKEY_FIELDS = [ 'id', 'userId', 'credentialId', 'publicKey', 'counter', 'transports', 'name', 'creationTime', 'lastUsedTime' ].join(','); const PASSKEY_FIELDS = [ 'id', 'userId', 'credentialId', 'publicKey', 'counter', 'transports', 'name', 'creationTime', 'lastUsedTime' ].join(',');
@@ -151,7 +151,7 @@ async function getRegistrationOptions(user) {
storeChallenge(user.id, options.challenge); storeChallenge(user.id, options.challenge);
debug(`getRegistrationOptions: generated for user ${user.id}`); log(`getRegistrationOptions: generated for user ${user.id}`);
return options; return options;
} }
@@ -179,7 +179,7 @@ async function verifyRegistration(user, response, name) {
})); }));
if (error) { if (error) {
debug(`verifyRegistration: verification failed for user ${user.id}:`, error); log(`verifyRegistration: verification failed for user ${user.id}:`, error);
throw new BoxError(BoxError.INVALID_CREDENTIALS, 'Passkey verification failed'); throw new BoxError(BoxError.INVALID_CREDENTIALS, 'Passkey verification failed');
} }
@@ -200,7 +200,7 @@ async function verifyRegistration(user, response, name) {
name || 'Passkey' name || 'Passkey'
); );
debug(`verifyRegistration: passkey registered for user ${user.id}`); log(`verifyRegistration: passkey registered for user ${user.id}`);
return result; return result;
} }
@@ -224,7 +224,7 @@ async function getAuthenticationOptions(user) {
storeChallenge(user.id, options.challenge); storeChallenge(user.id, options.challenge);
debug(`getAuthenticationOptions: generated for user ${user.id}`); log(`getAuthenticationOptions: generated for user ${user.id}`);
return options; return options;
} }
@@ -243,12 +243,12 @@ async function verifyAuthentication(user, response) {
const passkey = await getByCredentialId(credentialIdBase64url); const passkey = await getByCredentialId(credentialIdBase64url);
if (!passkey) { if (!passkey) {
debug(`verifyAuthentication: passkey not found for credential ${credentialIdBase64url}`); log(`verifyAuthentication: passkey not found for credential ${credentialIdBase64url}`);
throw new BoxError(BoxError.NOT_FOUND, 'Passkey not found'); throw new BoxError(BoxError.NOT_FOUND, 'Passkey not found');
} }
if (passkey.userId !== user.id) { if (passkey.userId !== user.id) {
debug(`verifyAuthentication: passkey belongs to different user`); log(`verifyAuthentication: passkey belongs to different user`);
throw new BoxError(BoxError.INVALID_CREDENTIALS, 'Passkey does not belong to this user'); throw new BoxError(BoxError.INVALID_CREDENTIALS, 'Passkey does not belong to this user');
} }
@@ -268,7 +268,7 @@ async function verifyAuthentication(user, response) {
})); }));
if (error) { if (error) {
debug(`verifyAuthentication: verification failed for user ${user.id}:`, error); log(`verifyAuthentication: verification failed for user ${user.id}:`, error);
throw new BoxError(BoxError.INVALID_CREDENTIALS, 'Passkey verification failed'); throw new BoxError(BoxError.INVALID_CREDENTIALS, 'Passkey verification failed');
} }
@@ -276,7 +276,7 @@ async function verifyAuthentication(user, response) {
await updateCounter(passkey.id, verification.authenticationInfo.newCounter); await updateCounter(passkey.id, verification.authenticationInfo.newCounter);
debug(`verifyAuthentication: passkey verified for user ${user.id}`); log(`verifyAuthentication: passkey verified for user ${user.id}`);
return { verified: true, passkeyId: passkey.id }; return { verified: true, passkeyId: passkey.id };
} }

64
src/platform.js
View File

@@ -7,7 +7,7 @@ import constants from './constants.js';
import cron from './cron.js'; import cron from './cron.js';
import dashboard from './dashboard.js'; import dashboard from './dashboard.js';
import database from './database.js'; import database from './database.js';
import debugModule from 'debug'; import logger from './logger.js';
import dockerProxy from './dockerproxy.js'; import dockerProxy from './dockerproxy.js';
import fs from 'node:fs'; import fs from 'node:fs';
import infra from './infra_version.js'; import infra from './infra_version.js';
@@ -26,7 +26,7 @@ import users from './users.js';
import volumes from './volumes.js'; import volumes from './volumes.js';
import _ from './underscore.js'; import _ from './underscore.js';
const debug = debugModule('box:platform'); const { log, trace } = logger('platform');
const shell = shellModule('platform'); const shell = shellModule('platform');
@@ -37,14 +37,14 @@ function getStatus() {
} }
async function pruneVolumes() { async function pruneVolumes() {
debug('pruneVolumes: remove all unused local volumes'); log('pruneVolumes: remove all unused local volumes');
const [error] = await safe(shell.spawn('docker', [ 'volume', 'prune', '--all', '--force' ], { encoding: 'utf8' })); const [error] = await safe(shell.spawn('docker', [ 'volume', 'prune', '--all', '--force' ], { encoding: 'utf8' }));
if (error) debug(`pruneVolumes: error pruning volumes: ${error.mesage}`); if (error) log(`pruneVolumes: error pruning volumes: ${error.mesage}`);
} }
async function createDockerNetwork() { async function createDockerNetwork() {
debug('createDockerNetwork: recreating docker network'); log('createDockerNetwork: recreating docker network');
await shell.spawn('docker', ['network', 'rm', '-f', 'cloudron'], {}); await shell.spawn('docker', ['network', 'rm', '-f', 'cloudron'], {});
// the --ipv6 option will work even in ipv6 is disabled. fd00 is IPv6 ULA // the --ipv6 option will work even in ipv6 is disabled. fd00 is IPv6 ULA
@@ -53,13 +53,13 @@ async function createDockerNetwork() {
} }
async function removeAllContainers() { async function removeAllContainers() {
debug('removeAllContainers: removing all containers for infra upgrade'); log('removeAllContainers: removing all containers for infra upgrade');
const output = await shell.spawn('docker', ['ps', '-qa', '--filter', 'label=isCloudronManaged'], { encoding: 'utf8' }); const output = await shell.spawn('docker', ['ps', '-qa', '--filter', 'label=isCloudronManaged'], { encoding: 'utf8' });
if (!output) return; if (!output) return;
for (const containerId of output.trim().split('\n')) { for (const containerId of output.trim().split('\n')) {
debug(`removeAllContainers: stopping and removing ${containerId}`); log(`removeAllContainers: stopping and removing ${containerId}`);
await shell.spawn('docker', ['stop', containerId], { encoding: 'utf8' }); await shell.spawn('docker', ['stop', containerId], { encoding: 'utf8' });
await shell.spawn('docker', ['rm', '-f', containerId], { encoding: 'utf8' }); await shell.spawn('docker', ['rm', '-f', containerId], { encoding: 'utf8' });
} }
@@ -70,10 +70,10 @@ async function markApps(existingInfra, restoreOptions) {
assert.strictEqual(typeof restoreOptions, 'object'); // { skipDnsSetup } assert.strictEqual(typeof restoreOptions, 'object'); // { skipDnsSetup }
if (existingInfra.version === 'none') { // cloudron is being restored from backup if (existingInfra.version === 'none') { // cloudron is being restored from backup
debug('markApps: restoring apps'); log('markApps: restoring apps');
await apps.restoreApps(await apps.list(), restoreOptions, AuditSource.PLATFORM); await apps.restoreApps(await apps.list(), restoreOptions, AuditSource.PLATFORM);
} else if (existingInfra.version !== infra.version) { } else if (existingInfra.version !== infra.version) {
debug('markApps: reconfiguring apps'); log('markApps: reconfiguring apps');
reverseProxy.removeAppConfigs(); // should we change the cert location, nginx will not start reverseProxy.removeAppConfigs(); // should we change the cert location, nginx will not start
await apps.configureApps(await apps.list(), { scheduleNow: false }, AuditSource.PLATFORM); // we will schedule it when infra is ready await apps.configureApps(await apps.list(), { scheduleNow: false }, AuditSource.PLATFORM); // we will schedule it when infra is ready
} else { } else {
@@ -85,25 +85,25 @@ async function markApps(existingInfra, restoreOptions) {
if (changedAddons.length) { if (changedAddons.length) {
// restart apps if docker image changes since the IP changes and any "persistent" connections fail // restart apps if docker image changes since the IP changes and any "persistent" connections fail
debug(`markApps: changedAddons: ${JSON.stringify(changedAddons)}`); log(`markApps: changedAddons: ${JSON.stringify(changedAddons)}`);
await apps.restartAppsUsingAddons(changedAddons, AuditSource.PLATFORM); await apps.restartAppsUsingAddons(changedAddons, AuditSource.PLATFORM);
} else { } else {
debug('markApps: apps are already uptodate'); log('markApps: apps are already uptodate');
} }
} }
} }
async function onInfraReady(infraChanged) { async function onInfraReady(infraChanged) {
debug(`onInfraReady: platform is ready. infra changed: ${infraChanged}`); log(`onInfraReady: platform is ready. infra changed: ${infraChanged}`);
gStatus.message = 'Ready'; gStatus.message = 'Ready';
gStatus.state = 'ready'; gStatus.state = 'ready';
if (infraChanged) await safe(pruneVolumes(), { debug }); // ignore error if (infraChanged) await safe(pruneVolumes(), { debug: log }); // ignore error
await apps.schedulePendingTasks(AuditSource.PLATFORM); await apps.schedulePendingTasks(AuditSource.PLATFORM);
await appTaskManager.start(); await appTaskManager.start();
// only prune services on infra change (which starts services for upgrade) // only prune services on infra change (which starts services for upgrade)
if (infraChanged) safe(services.stopUnusedServices(), { debug }); if (infraChanged) safe(services.stopUnusedServices(), { debug: log });
} }
async function startInfra(restoreOptions) { async function startInfra(restoreOptions) {
@@ -111,7 +111,7 @@ async function startInfra(restoreOptions) {
if (constants.TEST && !process.env.TEST_CREATE_INFRA) return; if (constants.TEST && !process.env.TEST_CREATE_INFRA) return;
debug('startInfra: checking infrastructure'); log('startInfra: checking infrastructure');
let existingInfra = { version: 'none' }; let existingInfra = { version: 'none' };
if (fs.existsSync(paths.INFRA_VERSION_FILE)) { if (fs.existsSync(paths.INFRA_VERSION_FILE)) {
@@ -121,13 +121,13 @@ async function startInfra(restoreOptions) {
// short-circuit for the restart case // short-circuit for the restart case
if (_.isEqual(infra, existingInfra)) { if (_.isEqual(infra, existingInfra)) {
debug('startInfra: infra is uptodate at version %s', infra.version); log('startInfra: infra is uptodate at version %s', infra.version);
safe(services.applyServiceLimits(), { debug }); safe(services.applyServiceLimits(), { debug: log });
await onInfraReady(false /* !infraChanged */); await onInfraReady(false /* !infraChanged */);
return; return;
} }
debug(`startInfra: updating infrastructure from ${existingInfra.version} to ${infra.version}`); log(`startInfra: updating infrastructure from ${existingInfra.version} to ${infra.version}`);
for (let attempt = 0; attempt < 5; attempt++) { for (let attempt = 0; attempt < 5; attempt++) {
try { try {
@@ -147,7 +147,7 @@ async function startInfra(restoreOptions) {
// for some reason, mysql arbitrary restarts making startup tasks fail. this makes the box update stuck // for some reason, mysql arbitrary restarts making startup tasks fail. this makes the box update stuck
// LOST is when existing connection breaks. REFUSED is when new connection cannot connect at all // LOST is when existing connection breaks. REFUSED is when new connection cannot connect at all
const retry = error.reason === BoxError.DATABASE_ERROR && (error.code === 'PROTOCOL_CONNECTION_LOST' || error.code === 'ECONNREFUSED'); const retry = error.reason === BoxError.DATABASE_ERROR && (error.code === 'PROTOCOL_CONNECTION_LOST' || error.code === 'ECONNREFUSED');
debug(`startInfra: Failed to start services. retry=${retry} (attempt ${attempt}): ${error}`); log(`startInfra: Failed to start services. retry=${retry} (attempt ${attempt}): ${error}`);
if (!retry) { if (!retry) {
gStatus.message = `Failed to start services. ${error.stdout ?? ''} ${error.stderr ?? ''}`; gStatus.message = `Failed to start services. ${error.stdout ?? ''} ${error.stderr ?? ''}`;
gStatus.state = 'failed'; gStatus.state = 'failed';
@@ -163,7 +163,7 @@ async function startInfra(restoreOptions) {
async function onActivated(restoreOptions) { async function onActivated(restoreOptions) {
assert.strictEqual(typeof restoreOptions, 'object'); // { skipDnsSetup } assert.strictEqual(typeof restoreOptions, 'object'); // { skipDnsSetup }
debug('onActivated: starting post activation services'); log('onActivated: starting post activation services');
// Starting the infra after a user is available means: // Starting the infra after a user is available means:
// 1. mail bounces can now be sent to the cloudron owner // 1. mail bounces can now be sent to the cloudron owner
@@ -177,11 +177,11 @@ async function onActivated(restoreOptions) {
if (!constants.TEST) await timers.setTimeout(30000); if (!constants.TEST) await timers.setTimeout(30000);
await reverseProxy.writeDefaultConfig({ activated :true }); await reverseProxy.writeDefaultConfig({ activated :true });
debug('onActivated: finished'); log('onActivated: finished');
} }
async function onDeactivated() { async function onDeactivated() {
debug('onDeactivated: stopping post activation services'); log('onDeactivated: stopping post activation services');
await cron.stopJobs(); await cron.stopJobs();
await dockerProxy.stop(); await dockerProxy.stop();
@@ -189,7 +189,7 @@ async function onDeactivated() {
} }
async function uninitialize() { async function uninitialize() {
debug('uninitializing platform'); log('uninitializing platform');
if (await users.isActivated()) await onDeactivated(); if (await users.isActivated()) await onDeactivated();
@@ -201,7 +201,7 @@ async function onDashboardLocationSet(subdomain, domain) {
assert.strictEqual(typeof subdomain, 'string'); assert.strictEqual(typeof subdomain, 'string');
assert.strictEqual(typeof domain, 'string'); assert.strictEqual(typeof domain, 'string');
await safe(reverseProxy.writeDashboardConfig(subdomain, domain), { debug }); // ok to fail if no disk space await safe(reverseProxy.writeDashboardConfig(subdomain, domain), { debug: log }); // ok to fail if no disk space
await oidcServer.stop(); await oidcServer.stop();
await oidcServer.start(); await oidcServer.start();
@@ -210,7 +210,7 @@ async function onDashboardLocationSet(subdomain, domain) {
} }
async function initialize() { async function initialize() {
debug('initialize: start platform'); log('initialize: start platform');
await database.initialize(); await database.initialize();
await tasks.stopAllTasks(); // when box code crashes, systemd will clean up the control-group but not the tasks await tasks.stopAllTasks(); // when box code crashes, systemd will clean up the control-group but not the tasks
@@ -225,13 +225,13 @@ async function initialize() {
// we remove the config as a simple security measure to not expose IP <-> domain // we remove the config as a simple security measure to not expose IP <-> domain
const activated = await users.isActivated(); const activated = await users.isActivated();
if (!activated) { if (!activated) {
debug('initialize: not activated. generating IP based redirection config'); log('initialize: not activated. generating IP based redirection config');
await safe(reverseProxy.writeDefaultConfig({ activated: false }), { debug }); // ok to fail if no disk space await safe(reverseProxy.writeDefaultConfig({ activated: false }), { debug: log }); // ok to fail if no disk space
} }
await updater.notifyBoxUpdate(); await updater.notifyBoxUpdate();
if (await users.isActivated()) safe(onActivated({ skipDnsSetup: false }), { debug }); // run in background if (await users.isActivated()) safe(onActivated({ skipDnsSetup: false }), { debug: log }); // run in background
} }
async function onDashboardLocationChanged(auditSource) { async function onDashboardLocationChanged(auditSource) {
@@ -239,9 +239,9 @@ async function onDashboardLocationChanged(auditSource) {
// mark all apps to be reconfigured, all have ExtraHosts injected // mark all apps to be reconfigured, all have ExtraHosts injected
const [, installedApps] = await safe(apps.list()); const [, installedApps] = await safe(apps.list());
await safe(apps.configureApps(installedApps, { scheduleNow: true }, auditSource), { debug }); await safe(apps.configureApps(installedApps, { scheduleNow: true }, auditSource), { debug: log });
await safe(services.rebuildService('turn', auditSource), { debug }); // to update the realm variable await safe(services.rebuildService('turn', auditSource), { debug: log }); // to update the realm variable
} }
async function onMailServerLocationChanged(auditSource) { async function onMailServerLocationChanged(auditSource) {
@@ -250,7 +250,7 @@ async function onMailServerLocationChanged(auditSource) {
// mark apps using email addon to be reconfigured // mark apps using email addon to be reconfigured
const [, installedApps] = await safe(apps.list()); const [, installedApps] = await safe(apps.list());
const appsUsingEmail = installedApps.filter((a) => !!a.manifest.addons?.email || a.manifest.addons?.sendmail?.requiresValidCertificate); const appsUsingEmail = installedApps.filter((a) => !!a.manifest.addons?.email || a.manifest.addons?.sendmail?.requiresValidCertificate);
await safe(apps.configureApps(appsUsingEmail, { scheduleNow: true }, auditSource), { debug }); await safe(apps.configureApps(appsUsingEmail, { scheduleNow: true }, auditSource), { debug: log });
} }
async function onMailServerIncomingDomainsChanged(auditSource) { async function onMailServerIncomingDomainsChanged(auditSource) {
@@ -259,7 +259,7 @@ async function onMailServerIncomingDomainsChanged(auditSource) {
// mark apps using email addon to be reconfigured // mark apps using email addon to be reconfigured
const [, installedApps] = await safe(apps.list()); const [, installedApps] = await safe(apps.list());
const appsUsingEmail = installedApps.filter((a) => !!a.manifest.addons?.email); const appsUsingEmail = installedApps.filter((a) => !!a.manifest.addons?.email);
await safe(apps.configureApps(appsUsingEmail, { scheduleNow: true }, auditSource), { debug }); await safe(apps.configureApps(appsUsingEmail, { scheduleNow: true }, auditSource), { debug: log });
} }
export default { export default {

30
src/provision.js
View File

@@ -6,7 +6,7 @@ import backuptask from './backuptask.js';
import BoxError from './boxerror.js'; import BoxError from './boxerror.js';
import dashboard from './dashboard.js'; import dashboard from './dashboard.js';
import constants from './constants.js'; import constants from './constants.js';
import debugModule from 'debug'; import logger from './logger.js';
import dns from './dns.js'; import dns from './dns.js';
import domains from './domains.js'; import domains from './domains.js';
import eventlog from './eventlog.js'; import eventlog from './eventlog.js';
@@ -27,7 +27,7 @@ import users from './users.js';
import tld from 'tldjs'; import tld from 'tldjs';
import tokens from './tokens.js'; import tokens from './tokens.js';
const debug = debugModule('box:provision'); const { log, trace } = logger('provision');
// we cannot use tasks since the tasks table gets overwritten when db is imported // we cannot use tasks since the tasks table gets overwritten when db is imported
@@ -51,13 +51,13 @@ const gStatus = {
}; };
function setProgress(task, message) { function setProgress(task, message) {
debug(`setProgress: ${task} - ${message}`); log(`setProgress: ${task} - ${message}`);
gStatus[task].message = message; gStatus[task].message = message;
} }
async function ensureDhparams() { async function ensureDhparams() {
if (fs.existsSync(paths.DHPARAMS_FILE)) return; if (fs.existsSync(paths.DHPARAMS_FILE)) return;
debug('ensureDhparams: generating dhparams'); log('ensureDhparams: generating dhparams');
const dhparams = await openssl.generateDhparam(); const dhparams = await openssl.generateDhparam();
if (!safe.fs.writeFileSync(paths.DHPARAMS_FILE, dhparams)) throw new BoxError(BoxError.FS_ERROR, `Could not save dhparams.pem: ${safe.error.message}`); if (!safe.fs.writeFileSync(paths.DHPARAMS_FILE, dhparams)) throw new BoxError(BoxError.FS_ERROR, `Could not save dhparams.pem: ${safe.error.message}`);
} }
@@ -75,7 +75,7 @@ async function setupTask(domain, auditSource) {
const location = { subdomain: constants.DASHBOARD_SUBDOMAIN, domain }; const location = { subdomain: constants.DASHBOARD_SUBDOMAIN, domain };
try { try {
debug(`setupTask: subdomain ${location.subdomain} and domain ${location.domain}`); log(`setupTask: subdomain ${location.subdomain} and domain ${location.domain}`);
await dns.registerLocations([location], { overwriteDns: true }, (progress) => setProgress('setup', progress.message)); await dns.registerLocations([location], { overwriteDns: true }, (progress) => setProgress('setup', progress.message));
await dns.waitForLocations([location], (progress) => setProgress('setup', progress.message)); await dns.waitForLocations([location], (progress) => setProgress('setup', progress.message));
await reverseProxy.ensureCertificate(location, {}, auditSource); await reverseProxy.ensureCertificate(location, {}, auditSource);
@@ -85,7 +85,7 @@ async function setupTask(domain, auditSource) {
setProgress('setup', 'Done'), setProgress('setup', 'Done'),
await eventlog.add(eventlog.ACTION_PROVISION, auditSource, {}); await eventlog.add(eventlog.ACTION_PROVISION, auditSource, {});
} catch (error) { } catch (error) {
debug('setupTask: error. %o', error); log('setupTask: error. %o', error);
gStatus.setup.errorMessage = error.message; gStatus.setup.errorMessage = error.message;
} }
@@ -111,7 +111,7 @@ async function setup(domainConfig, ipv4Config, ipv6Config, auditSource) {
const domain = domainConfig.domain.toLowerCase(); const domain = domainConfig.domain.toLowerCase();
const zoneName = domainConfig.zoneName ? domainConfig.zoneName : (tld.getDomain(domain) || domain); const zoneName = domainConfig.zoneName ? domainConfig.zoneName : (tld.getDomain(domain) || domain);
debug(`setup: domain ${domain} and zone ${zoneName}`); log(`setup: domain ${domain} and zone ${zoneName}`);
const data = { const data = {
zoneName: zoneName, zoneName: zoneName,
@@ -127,9 +127,9 @@ async function setup(domainConfig, ipv4Config, ipv6Config, auditSource) {
await network.setIPv4Config(ipv4Config); await network.setIPv4Config(ipv4Config);
await network.setIPv6Config(ipv6Config); await network.setIPv6Config(ipv6Config);
safe(setupTask(domain, auditSource), { debug }); // now that args are validated run the task in the background safe(setupTask(domain, auditSource), { debug: log }); // now that args are validated run the task in the background
} catch (error) { } catch (error) {
debug('setup: error. %o', error); log('setup: error. %o', error);
gStatus.setup.active = false; gStatus.setup.active = false;
gStatus.setup.errorMessage = error.message; gStatus.setup.errorMessage = error.message;
throw error; throw error;
@@ -144,7 +144,7 @@ async function activate(username, password, email, displayName, ip, auditSource)
assert.strictEqual(typeof ip, 'string'); assert.strictEqual(typeof ip, 'string');
assert.strictEqual(typeof auditSource, 'object'); assert.strictEqual(typeof auditSource, 'object');
debug(`activate: user: ${username} email:${email}`); log(`activate: user: ${username} email:${email}`);
await appstore.registerCloudron3(); await appstore.registerCloudron3();
@@ -157,7 +157,7 @@ async function activate(username, password, email, displayName, ip, auditSource)
await eventlog.add(eventlog.ACTION_ACTIVATE, auditSource, {}); await eventlog.add(eventlog.ACTION_ACTIVATE, auditSource, {});
safe(platform.onActivated({ skipDnsSetup: false }), { debug }); // background safe(platform.onActivated({ skipDnsSetup: false }), { debug: log }); // background
return { return {
userId: ownerId, userId: ownerId,
@@ -215,11 +215,11 @@ async function restoreTask(backupSite, remotePath, ipv4Config, ipv6Config, optio
await appstore.checkSubscription(); // never throws. worst case, user has to visit the Account view to refresh subscription info await appstore.checkSubscription(); // never throws. worst case, user has to visit the Account view to refresh subscription info
safe(platform.onActivated({ skipDnsSetup: options.skipDnsSetup }), { debug }); // background safe(platform.onActivated({ skipDnsSetup: options.skipDnsSetup }), { debug: log }); // background
await backupSites.storageApi(backupSite).teardown(backupSite.config); await backupSites.storageApi(backupSite).teardown(backupSite.config);
} catch (error) { } catch (error) {
debug('restoreTask: error. %o', error); log('restoreTask: error. %o', error);
gStatus.restore.errorMessage = error ? error.message : ''; gStatus.restore.errorMessage = error ? error.message : '';
} }
gStatus.restore.active = false; gStatus.restore.active = false;
@@ -260,9 +260,9 @@ async function restore(backupConfig, remotePath, version, ipv4Config, ipv6Config
const ipv6Error = await network.testIPv6Config(ipv6Config); const ipv6Error = await network.testIPv6Config(ipv6Config);
if (ipv6Error) throw ipv6Error; if (ipv6Error) throw ipv6Error;
safe(restoreTask(backupSite, remotePath, ipv4Config, ipv6Config, options, auditSource), { debug }); // now that args are validated run the task in the background safe(restoreTask(backupSite, remotePath, ipv4Config, ipv6Config, options, auditSource), { debug: log }); // now that args are validated run the task in the background
} catch (error) { } catch (error) {
debug('restore: error. %o', error); log('restore: error. %o', error);
gStatus.restore.active = false; gStatus.restore.active = false;
gStatus.restore.errorMessage = error.message; gStatus.restore.errorMessage = error.message;
throw error; throw error;

10
src/proxyauth.js
View File

@@ -3,7 +3,7 @@ import assert from 'node:assert';
import blobs from './blobs.js'; import blobs from './blobs.js';
import constants from './constants.js'; import constants from './constants.js';
import dashboard from './dashboard.js'; import dashboard from './dashboard.js';
import debugModule from 'debug'; import logger from './logger.js';
import ejs from 'ejs'; import ejs from 'ejs';
import express from 'express'; import express from 'express';
import fs from 'node:fs'; import fs from 'node:fs';
@@ -21,7 +21,7 @@ import settings from './settings.js';
import users from './users.js'; import users from './users.js';
import util from 'node:util'; import util from 'node:util';
const debug = debugModule('box:proxyAuth'); const { log, trace } = logger('proxyAuth');
// heavily inspired from https://gock.net/blog/2020/nginx-subrequest-authentication-server/ and https://github.com/andygock/auth-server // heavily inspired from https://gock.net/blog/2020/nginx-subrequest-authentication-server/ and https://github.com/andygock/auth-server
@@ -36,7 +36,7 @@ function jwtVerify(req, res, next) {
jwt.verify(token, gTokenSecret, function (error, decoded) { jwt.verify(token, gTokenSecret, function (error, decoded) {
if (error) { if (error) {
debug('jwtVerify: malformed token or bad signature', error.message); log('jwtVerify: malformed token or bad signature', error.message);
req.user = null; req.user = null;
} else { } else {
req.user = decoded.user || null; req.user = decoded.user || null;
@@ -160,7 +160,7 @@ async function login(req, res, next) {
async function callback(req, res, next) { async function callback(req, res, next) {
if (typeof req.query.code !== 'string') return next(new HttpError(400, 'missing query argument "code"')); if (typeof req.query.code !== 'string') return next(new HttpError(400, 'missing query argument "code"'));
debug(`callback: with code ${req.query.code}`); log(`callback: with code ${req.query.code}`);
const username = await oidcServer.consumeAuthCode(req.query.code); const username = await oidcServer.consumeAuthCode(req.query.code);
if (!username) return next(new HttpError(400, 'invalid "code"')); if (!username) return next(new HttpError(400, 'invalid "code"'));
@@ -237,7 +237,7 @@ async function start() {
gTokenSecret = await blobs.getString(blobs.PROXY_AUTH_TOKEN_SECRET); gTokenSecret = await blobs.getString(blobs.PROXY_AUTH_TOKEN_SECRET);
if (!gTokenSecret) { if (!gTokenSecret) {
debug('start: generating new token secret'); log('start: generating new token secret');
gTokenSecret = hat(64); gTokenSecret = hat(64);
await blobs.setString(blobs.PROXY_AUTH_TOKEN_SECRET, gTokenSecret); await blobs.setString(blobs.PROXY_AUTH_TOKEN_SECRET, gTokenSecret);
} }

58
src/reverseproxy.js
View File

@@ -5,7 +5,7 @@ import blobs from './blobs.js';
import BoxError from './boxerror.js'; import BoxError from './boxerror.js';
import constants from './constants.js'; import constants from './constants.js';
import dashboard from './dashboard.js'; import dashboard from './dashboard.js';
import debugModule from 'debug'; import logger from './logger.js';
import dns from './dns.js'; import dns from './dns.js';
import docker from './docker.js'; import docker from './docker.js';
import domains from './domains.js'; import domains from './domains.js';
@@ -24,7 +24,7 @@ import settings from './settings.js';
import shellModule from './shell.js'; import shellModule from './shell.js';
import tasks from './tasks.js'; import tasks from './tasks.js';
const debug = debugModule('box:reverseproxy'); const { log, trace } = logger('reverseproxy');
const shell = shellModule('reverseproxy'); const shell = shellModule('reverseproxy');
const NGINX_APPCONFIG_EJS = fs.readFileSync(import.meta.dirname + '/nginxconfig.ejs', { encoding: 'utf8' }); const NGINX_APPCONFIG_EJS = fs.readFileSync(import.meta.dirname + '/nginxconfig.ejs', { encoding: 'utf8' });
@@ -59,7 +59,7 @@ async function providerMatches(domainObject, cert) {
const mismatch = issuerMismatch || wildcardMismatch; const mismatch = issuerMismatch || wildcardMismatch;
debug(`providerMatches: subject=${subject} domain=${domain} issuer=${issuer} ` log(`providerMatches: subject=${subject} domain=${domain} issuer=${issuer} `
+ `wildcard=${isWildcardCert}/${wildcard} prod=${isLetsEncryptProd}/${prod} ` + `wildcard=${isWildcardCert}/${wildcard} prod=${isLetsEncryptProd}/${prod} `
+ `issuerMismatch=${issuerMismatch} wildcardMismatch=${wildcardMismatch} match=${!mismatch}`); + `issuerMismatch=${issuerMismatch} wildcardMismatch=${wildcardMismatch} match=${!mismatch}`);
@@ -110,7 +110,7 @@ async function needsRenewal(cert, renewalInfo, options) {
let renew = false; let renew = false;
if (now.getTime() >= rt.getTime()) renew = true; // renew immediately since now is in the past if (now.getTime() >= rt.getTime()) renew = true; // renew immediately since now is in the past
else if ((now.getTime() + (24*60*60*1000)) >= rt.getTime()) renew = true; // next cron run will be in the past else if ((now.getTime() + (24*60*60*1000)) >= rt.getTime()) renew = true; // next cron run will be in the past
debug(`needsRenewal: ${renew}. ARI ${JSON.stringify(renewalInfo)}`); log(`needsRenewal: ${renew}. ARI ${JSON.stringify(renewalInfo)}`);
return renew; // can wait return renew; // can wait
} }
@@ -122,7 +122,7 @@ async function needsRenewal(cert, renewalInfo, options) {
isExpiring = (endDate - now) <= (30 * 24 * 60 * 60 * 1000); // expiring in a month isExpiring = (endDate - now) <= (30 * 24 * 60 * 60 * 1000); // expiring in a month
} }
debug(`needsRenewal: ${isExpiring}. force: ${!!options.forceRenewal}`); log(`needsRenewal: ${isExpiring}. force: ${!!options.forceRenewal}`);
return isExpiring; return isExpiring;
} }
@@ -185,7 +185,7 @@ async function setupTlsAddon(app) {
for (const content of contents) { for (const content of contents) {
if (writeFileSync(`${certificateDir}/${content.filename}`, content.data)) ++changed; if (writeFileSync(`${certificateDir}/${content.filename}`, content.data)) ++changed;
} }
debug(`setupTlsAddon: ${changed} files changed`); log(`setupTlsAddon: ${changed} files changed`);
// clean up any certs of old locations // clean up any certs of old locations
const filenamesInUse = new Set(contents.map(c => c.filename)); const filenamesInUse = new Set(contents.map(c => c.filename));
@@ -196,7 +196,7 @@ async function setupTlsAddon(app) {
safe.fs.unlinkSync(path.join(certificateDir, filename)); safe.fs.unlinkSync(path.join(certificateDir, filename));
++removed; ++removed;
} }
debug(`setupTlsAddon: ${removed} files removed`); log(`setupTlsAddon: ${removed} files removed`);
if (changed || removed) await docker.restartContainer(app.id); if (changed || removed) await docker.restartContainer(app.id);
} }
@@ -215,7 +215,7 @@ async function setFallbackCertificate(domain, certificate) {
assert.strictEqual(typeof domain, 'string'); assert.strictEqual(typeof domain, 'string');
assert(certificate && typeof certificate === 'object'); assert(certificate && typeof certificate === 'object');
debug(`setFallbackCertificate: setting certs for domain ${domain}`); log(`setFallbackCertificate: setting certs for domain ${domain}`);
if (!safe.fs.writeFileSync(path.join(paths.NGINX_CERT_DIR, `${domain}.host.cert`), certificate.cert)) throw new BoxError(BoxError.FS_ERROR, safe.error.message); if (!safe.fs.writeFileSync(path.join(paths.NGINX_CERT_DIR, `${domain}.host.cert`), certificate.cert)) throw new BoxError(BoxError.FS_ERROR, safe.error.message);
if (!safe.fs.writeFileSync(path.join(paths.NGINX_CERT_DIR, `${domain}.host.key`), certificate.key)) throw new BoxError(BoxError.FS_ERROR, safe.error.message); if (!safe.fs.writeFileSync(path.join(paths.NGINX_CERT_DIR, `${domain}.host.key`), certificate.key)) throw new BoxError(BoxError.FS_ERROR, safe.error.message);
@@ -245,7 +245,7 @@ async function writeCertificate(location) {
const certFilePath = path.join(paths.NGINX_CERT_DIR, `${domain}.host.cert`); const certFilePath = path.join(paths.NGINX_CERT_DIR, `${domain}.host.cert`);
const keyFilePath = path.join(paths.NGINX_CERT_DIR, `${domain}.host.key`); const keyFilePath = path.join(paths.NGINX_CERT_DIR, `${domain}.host.key`);
debug(`writeCertificate: ${fqdn} will use fallback certs`); log(`writeCertificate: ${fqdn} will use fallback certs`);
writeFileSync(certFilePath, domainObject.fallbackCertificate.cert); writeFileSync(certFilePath, domainObject.fallbackCertificate.cert);
writeFileSync(keyFilePath, domainObject.fallbackCertificate.key); writeFileSync(keyFilePath, domainObject.fallbackCertificate.key);
@@ -257,7 +257,7 @@ async function writeCertificate(location) {
let key = await blobs.getString(`${blobs.CERT_PREFIX}-${certName}.key`); let key = await blobs.getString(`${blobs.CERT_PREFIX}-${certName}.key`);
if (!key || !cert) { // use fallback certs if we didn't manage to get acme certs if (!key || !cert) { // use fallback certs if we didn't manage to get acme certs
debug(`writeCertificate: ${fqdn} will use fallback certs because acme is missing`); log(`writeCertificate: ${fqdn} will use fallback certs because acme is missing`);
cert = domainObject.fallbackCertificate.cert; cert = domainObject.fallbackCertificate.cert;
key = domainObject.fallbackCertificate.key; key = domainObject.fallbackCertificate.key;
} }
@@ -276,7 +276,7 @@ async function getKey(certName) {
const key = await blobs.getString(`${blobs.CERT_PREFIX}-${certName}.key`); const key = await blobs.getString(`${blobs.CERT_PREFIX}-${certName}.key`);
if (key) return key; if (key) return key;
debug(`ensureKey: generating new key for ${certName}`); log(`ensureKey: generating new key for ${certName}`);
// secp384r1 is same as prime256v1. openssl ecparam -list_curves. we used to use secp384r1 but it doesn't seem to be accepted by few mail servers // secp384r1 is same as prime256v1. openssl ecparam -list_curves. we used to use secp384r1 but it doesn't seem to be accepted by few mail servers
return await openssl.generateKey('secp256r1'); return await openssl.generateKey('secp256r1');
}; };
@@ -287,13 +287,13 @@ async function getRenewalInfo(cert, certName) {
if (Date.now() < (new Date(renewalInfo.valid)).getTime()) return renewalInfo; // still valid if (Date.now() < (new Date(renewalInfo.valid)).getTime()) return renewalInfo; // still valid
debug(`getRenewalInfo: ${certName} refreshing`); log(`getRenewalInfo: ${certName} refreshing`);
const [error, result] = await safe(acme2.getRenewalInfo(cert, renewalInfo.url)); const [error, result] = await safe(acme2.getRenewalInfo(cert, renewalInfo.url));
if (error) { if (error) {
debug(`getRenewalInfo: ${certName} error getting renewal info`, error); log(`getRenewalInfo: ${certName} error getting renewal info`, error);
await blobs.del(`${blobs.CERT_PREFIX}-${certName}.renewal`); await blobs.del(`${blobs.CERT_PREFIX}-${certName}.renewal`);
} else { } else {
debug(`getRenewalInfo: ${certName} updated: ${JSON.stringify(result)}`); log(`getRenewalInfo: ${certName} updated: ${JSON.stringify(result)}`);
await blobs.setString(`${blobs.CERT_PREFIX}-${certName}.renewal`, JSON.stringify(result)); await blobs.setString(`${blobs.CERT_PREFIX}-${certName}.renewal`, JSON.stringify(result));
} }
@@ -311,12 +311,12 @@ async function ensureCertificate(location, options, auditSource) {
const fqdn = dns.fqdn(location.subdomain, location.domain); const fqdn = dns.fqdn(location.subdomain, location.domain);
if (location.certificate) { // user certificate if (location.certificate) { // user certificate
debug(`ensureCertificate: ${fqdn} will use user certs`); log(`ensureCertificate: ${fqdn} will use user certs`);
return; return;
} }
if (domainObject.tlsConfig.provider === 'fallback') { if (domainObject.tlsConfig.provider === 'fallback') {
debug(`ensureCertificate: ${fqdn} will use fallback certs`); log(`ensureCertificate: ${fqdn} will use fallback certs`);
return; return;
} }
@@ -330,13 +330,13 @@ async function ensureCertificate(location, options, auditSource) {
const outdated = await needsRenewal(cert, renewalInfo, options); const outdated = await needsRenewal(cert, renewalInfo, options);
if (sameProvider && !outdated) { if (sameProvider && !outdated) {
debug(`ensureCertificate: ${fqdn} acme cert exists and is up to date`); log(`ensureCertificate: ${fqdn} acme cert exists and is up to date`);
return; return;
} }
debug(`ensureCertificate: ${fqdn} acme cert exists but provider mismatch or needs renewal`); log(`ensureCertificate: ${fqdn} acme cert exists but provider mismatch or needs renewal`);
} }
debug(`ensureCertificate: ${fqdn} needs acme cert`); log(`ensureCertificate: ${fqdn} needs acme cert`);
const [error, result] = await safe(acme2.getCertificate(fqdn, domainObject, key)); const [error, result] = await safe(acme2.getCertificate(fqdn, domainObject, key));
if (!error) { if (!error) {
@@ -346,7 +346,7 @@ async function ensureCertificate(location, options, auditSource) {
await blobs.setString(`${blobs.CERT_PREFIX}-${certName}.renewal`, JSON.stringify(result.renewalInfo)); await blobs.setString(`${blobs.CERT_PREFIX}-${certName}.renewal`, JSON.stringify(result.renewalInfo));
} }
debug(`ensureCertificate: error: ${error?.message || 'null'}`); log(`ensureCertificate: error: ${error?.message || 'null'}`);
await safe(eventlog.add(eventlog.ACTION_CERTIFICATE_NEW, auditSource, { domain: fqdn, errorMessage: error?.message || '', renewalInfo: result?.renewalInfo || null })); await safe(eventlog.add(eventlog.ACTION_CERTIFICATE_NEW, auditSource, { domain: fqdn, errorMessage: error?.message || '', renewalInfo: result?.renewalInfo || null }));
} }
@@ -377,7 +377,7 @@ async function writeDashboardConfig(subdomain, domain) {
assert.strictEqual(typeof subdomain, 'string'); assert.strictEqual(typeof subdomain, 'string');
assert.strictEqual(typeof domain, 'string'); assert.strictEqual(typeof domain, 'string');
debug(`writeDashboardConfig: writing dashboard config for ${domain}`); log(`writeDashboardConfig: writing dashboard config for ${domain}`);
const dashboardFqdn = dns.fqdn(subdomain, domain); const dashboardFqdn = dns.fqdn(subdomain, domain);
const location = { domain, fqdn: dashboardFqdn, certificate: null }; const location = { domain, fqdn: dashboardFqdn, certificate: null };
@@ -390,7 +390,7 @@ async function removeDashboardConfig(subdomain, domain) {
assert.strictEqual(typeof subdomain, 'string'); assert.strictEqual(typeof subdomain, 'string');
assert.strictEqual(typeof domain, 'string'); assert.strictEqual(typeof domain, 'string');
debug(`removeDashboardConfig: removing dashboard config of ${domain}`); log(`removeDashboardConfig: removing dashboard config of ${domain}`);
const vhost = dns.fqdn(subdomain, domain); const vhost = dns.fqdn(subdomain, domain);
const nginxConfigFilename = path.join(paths.NGINX_APPCONFIG_DIR, `dashboard/${vhost}.conf`); const nginxConfigFilename = path.join(paths.NGINX_APPCONFIG_DIR, `dashboard/${vhost}.conf`);
@@ -473,7 +473,7 @@ async function writeAppLocationNginxConfig(app, location, certificatePath) {
const nginxConf = ejs.render(NGINX_APPCONFIG_EJS, data); const nginxConf = ejs.render(NGINX_APPCONFIG_EJS, data);
const filename = path.join(paths.NGINX_APPCONFIG_DIR, app.id, `${fqdn.replace('*', '_')}.conf`); const filename = path.join(paths.NGINX_APPCONFIG_DIR, app.id, `${fqdn.replace('*', '_')}.conf`);
debug(`writeAppLocationNginxConfig: writing config for "${fqdn}" to ${filename} with options ${JSON.stringify(data)}`); log(`writeAppLocationNginxConfig: writing config for "${fqdn}" to ${filename} with options ${JSON.stringify(data)}`);
writeFileSync(filename, nginxConf); writeFileSync(filename, nginxConf);
} }
@@ -566,7 +566,7 @@ async function cleanupCerts(locations, auditSource, progressCallback) {
if (removedCertNames.length) await safe(eventlog.add(eventlog.ACTION_CERTIFICATE_CLEANUP, auditSource, { domains: removedCertNames })); if (removedCertNames.length) await safe(eventlog.add(eventlog.ACTION_CERTIFICATE_CLEANUP, auditSource, { domains: removedCertNames }));
debug('cleanupCerts: done'); log('cleanupCerts: done');
} }
async function checkCerts(options, auditSource, progressCallback) { async function checkCerts(options, auditSource, progressCallback) {
@@ -621,12 +621,12 @@ async function startRenewCerts(options, auditSource) {
assert.strictEqual(typeof auditSource, 'object'); assert.strictEqual(typeof auditSource, 'object');
const taskId = await tasks.add(tasks.TASK_CHECK_CERTS, [ options, auditSource ]); const taskId = await tasks.add(tasks.TASK_CHECK_CERTS, [ options, auditSource ]);
safe(tasks.startTask(taskId, {}), { debug }); // background safe(tasks.startTask(taskId, {}), { debug: log }); // background
return taskId; return taskId;
} }
function removeAppConfigs() { function removeAppConfigs() {
debug('removeAppConfigs: removing app nginx configs'); log('removeAppConfigs: removing app nginx configs');
// remove all configs which are not the default or current dashboard // remove all configs which are not the default or current dashboard
for (const entry of fs.readdirSync(paths.NGINX_APPCONFIG_DIR, { withFileTypes: true })) { for (const entry of fs.readdirSync(paths.NGINX_APPCONFIG_DIR, { withFileTypes: true })) {
@@ -649,7 +649,7 @@ async function writeDefaultConfig(options) {
const keyFilePath = path.join(paths.NGINX_CERT_DIR, 'default.key'); const keyFilePath = path.join(paths.NGINX_CERT_DIR, 'default.key');
if (!fs.existsSync(certFilePath) || !fs.existsSync(keyFilePath)) { if (!fs.existsSync(certFilePath) || !fs.existsSync(keyFilePath)) {
debug('writeDefaultConfig: create new cert'); log('writeDefaultConfig: create new cert');
const cn = 'cloudron-' + (new Date()).toISOString(); // randomize date a bit to keep firefox happy const cn = 'cloudron-' + (new Date()).toISOString(); // randomize date a bit to keep firefox happy
@@ -672,7 +672,7 @@ async function writeDefaultConfig(options) {
const nginxConf = ejs.render(NGINX_APPCONFIG_EJS, data); const nginxConf = ejs.render(NGINX_APPCONFIG_EJS, data);
const nginxConfigFilename = path.join(paths.NGINX_APPCONFIG_DIR, constants.NGINX_DEFAULT_CONFIG_FILE_NAME); const nginxConfigFilename = path.join(paths.NGINX_APPCONFIG_DIR, constants.NGINX_DEFAULT_CONFIG_FILE_NAME);
debug(`writeDefaultConfig: writing configs for endpoint "${data.endpoint}"`); log(`writeDefaultConfig: writing configs for endpoint "${data.endpoint}"`);
if (!safe.fs.writeFileSync(nginxConfigFilename, nginxConf)) throw new BoxError(BoxError.FS_ERROR, safe.error); if (!safe.fs.writeFileSync(nginxConfigFilename, nginxConf)) throw new BoxError(BoxError.FS_ERROR, safe.error);
@@ -708,7 +708,7 @@ async function setTrustedIps(trustedIps) {
} }
async function reprovision() { async function reprovision() {
debug('reprovision: restoring fallback certs and trusted ips'); log('reprovision: restoring fallback certs and trusted ips');
const result = await domains.list(); const result = await domains.list();

6
src/routes/accesscontrol.js
View File

@@ -1,13 +1,13 @@
import apps from '../apps.js'; import apps from '../apps.js';
import assert from 'node:assert'; import assert from 'node:assert';
import BoxError from '../boxerror.js'; import BoxError from '../boxerror.js';
import debugModule from 'debug'; import logger from '../logger.js';
import { HttpError } from '@cloudron/connect-lastmile'; import { HttpError } from '@cloudron/connect-lastmile';
import safe from 'safetydance'; import safe from 'safetydance';
import tokens from '../tokens.js'; import tokens from '../tokens.js';
import users from '../users.js'; import users from '../users.js';
const debug = debugModule('box:routes/accesscontrol'); const { log, trace } = logger('routes/accesscontrol');
async function passwordAuth(req, res, next) { async function passwordAuth(req, res, next) {
@@ -55,7 +55,7 @@ async function tokenAuth(req, res, next) {
const user = await users.get(token.identifier); const user = await users.get(token.identifier);
if (!user) return next(new HttpError(401, 'User not found')); if (!user) return next(new HttpError(401, 'User not found'));
if (!user.active) { if (!user.active) {
debug(`tokenAuth: ${user.username || user.id} is not active`); log(`tokenAuth: ${user.username || user.id} is not active`);
return next(new HttpError(401, 'User not active')); return next(new HttpError(401, 'User not active'));
} }

8
src/routes/apps.js
View File

@@ -8,7 +8,7 @@ import backupSites from '../backupsites.js';
import BoxError from '../boxerror.js'; import BoxError from '../boxerror.js';
import community from '../community.js'; import community from '../community.js';
import constants from '../constants.js'; import constants from '../constants.js';
import debugModule from 'debug'; import logger from '../logger.js';
import { HttpError } from '@cloudron/connect-lastmile'; import { HttpError } from '@cloudron/connect-lastmile';
import { HttpSuccess } from '@cloudron/connect-lastmile'; import { HttpSuccess } from '@cloudron/connect-lastmile';
import metrics from '../metrics.js'; import metrics from '../metrics.js';
@@ -18,7 +18,7 @@ import users from '../users.js';
import { getImageContentType } from '../image-content-type.js'; import { getImageContentType } from '../image-content-type.js';
import WebSocket from 'ws'; import WebSocket from 'ws';
const debug = debugModule('box:routes/apps'); const { log, trace } = logger('routes/apps');
async function load(req, res, next) { async function load(req, res, next) {
@@ -829,7 +829,7 @@ async function startExecWebSocket(req, res, next) {
duplexStream.on('end', function () { ws.close(); }); duplexStream.on('end', function () { ws.close(); });
duplexStream.on('close', function () { ws.close(); }); duplexStream.on('close', function () { ws.close(); });
duplexStream.on('error', function (streamError) { duplexStream.on('error', function (streamError) {
debug('duplexStream error: %o', streamError); log('duplexStream error: %o', streamError);
}); });
duplexStream.on('data', function (data) { duplexStream.on('data', function (data) {
if (ws.readyState !== WebSocket.OPEN) return; if (ws.readyState !== WebSocket.OPEN) return;
@@ -837,7 +837,7 @@ async function startExecWebSocket(req, res, next) {
}); });
ws.on('error', function (wsError) { ws.on('error', function (wsError) {
debug('websocket error: %o', wsError); log('websocket error: %o', wsError);
}); });
ws.on('message', function (msg) { ws.on('message', function (msg) {
duplexStream.write(msg); duplexStream.write(msg);

6
src/routes/auth.js
View File

@@ -2,7 +2,7 @@ import assert from 'node:assert';
import AuditSource from '../auditsource.js'; import AuditSource from '../auditsource.js';
import BoxError from '../boxerror.js'; import BoxError from '../boxerror.js';
import constants from '../constants.js'; import constants from '../constants.js';
import debugModule from 'debug'; import logger from '../logger.js';
import eventlog from '../eventlog.js'; import eventlog from '../eventlog.js';
import { HttpError } from '@cloudron/connect-lastmile'; import { HttpError } from '@cloudron/connect-lastmile';
import { HttpSuccess } from '@cloudron/connect-lastmile'; import { HttpSuccess } from '@cloudron/connect-lastmile';
@@ -12,7 +12,7 @@ import speakeasy from 'speakeasy';
import tokens from '../tokens.js'; import tokens from '../tokens.js';
import users from '../users.js'; import users from '../users.js';
const debug = debugModule('box:routes/cloudron'); const { log, trace } = logger('routes/cloudron');
async function login(req, res, next) { async function login(req, res, next) {
@@ -32,7 +32,7 @@ async function login(req, res, next) {
const auditSource = AuditSource.fromRequest(req); const auditSource = AuditSource.fromRequest(req);
await eventlog.add(req.user.ghost ? eventlog.ACTION_USER_LOGIN_GHOST : eventlog.ACTION_USER_LOGIN, auditSource, { userId: req.user.id, user: users.removePrivateFields(req.user), type, appId: oidcClients.ID_CLI }); await eventlog.add(req.user.ghost ? eventlog.ACTION_USER_LOGIN_GHOST : eventlog.ACTION_USER_LOGIN, auditSource, { userId: req.user.id, user: users.removePrivateFields(req.user), type, appId: oidcClients.ID_CLI });
await safe(users.notifyLoginLocation(req.user, ip, userAgent, auditSource), { debug }); await safe(users.notifyLoginLocation(req.user, ip, userAgent, auditSource), { debug: log });
next(new HttpSuccess(200, token)); next(new HttpSuccess(200, token));
} }

6
src/routes/mailserver.js
View File

@@ -1,7 +1,7 @@
import assert from 'node:assert'; import assert from 'node:assert';
import AuditSource from '../auditsource.js'; import AuditSource from '../auditsource.js';
import BoxError from '../boxerror.js'; import BoxError from '../boxerror.js';
import debugModule from 'debug'; import logger from '../logger.js';
import http from 'node:http'; import http from 'node:http';
import { HttpError } from '@cloudron/connect-lastmile'; import { HttpError } from '@cloudron/connect-lastmile';
import { HttpSuccess } from '@cloudron/connect-lastmile'; import { HttpSuccess } from '@cloudron/connect-lastmile';
@@ -9,7 +9,7 @@ import mailServer from '../mailserver.js';
import safe from 'safetydance'; import safe from 'safetydance';
import services from '../services.js'; import services from '../services.js';
const debug = debugModule('box:routes/mailserver'); const { log, trace } = logger('routes/mailserver');
async function proxyToMailContainer(port, pathname, req, res, next) { async function proxyToMailContainer(port, pathname, req, res, next) {
@@ -61,7 +61,7 @@ async function proxyAndRestart(req, res, next) {
if (httpError) return next(httpError); if (httpError) return next(httpError);
// for success, the proxy already sent the response. do not proceed to connect-lastmile which will result in double headers // for success, the proxy already sent the response. do not proceed to connect-lastmile which will result in double headers
await safe(mailServer.restart(), { debug }); await safe(mailServer.restart(), { debug: log });
}); });
} }

26
src/routes/test/common.js
View File

@@ -1,7 +1,7 @@
import apps from '../../apps.js'; import apps from '../../apps.js';
import appstore from '../../appstore.js'; import appstore from '../../appstore.js';
import backupSites from '../../backupsites.js'; import backupSites from '../../backupsites.js';
import debugModule from 'debug'; import logger from '../../logger.js';
import constants from '../../constants.js'; import constants from '../../constants.js';
import database from '../../database.js'; import database from '../../database.js';
import assert from 'node:assert/strict'; import assert from 'node:assert/strict';
@@ -16,7 +16,7 @@ import tasks from '../../tasks.js';
import timers from 'timers/promises'; import timers from 'timers/promises';
import tokens from '../../tokens.js'; import tokens from '../../tokens.js';
const debug = debugModule('box:test/common'); const { log, trace } = logger('test/common');
const manifest = { const manifest = {
'id': 'io.cloudron.test', 'id': 'io.cloudron.test',
@@ -98,17 +98,17 @@ const serverUrl = `http://localhost:${constants.PORT}`;
async function setupServer() { async function setupServer() {
debug('Setting up server'); log('Setting up server');
await database.initialize(); await database.initialize();
await database._clear(); await database._clear();
await appstore._setApiServerOrigin(mockApiServerOrigin); await appstore._setApiServerOrigin(mockApiServerOrigin);
await oidcServer.stop(); await oidcServer.stop();
await server.start(); await server.start();
debug('Set up server complete'); log('Set up server complete');
} }
async function setup() { async function setup() {
debug('Setting up'); log('Setting up');
await setupServer(); await setupServer();
@@ -165,15 +165,15 @@ async function setup() {
await settings._set(settings.APPSTORE_API_TOKEN_KEY, appstoreToken); // appstore token await settings._set(settings.APPSTORE_API_TOKEN_KEY, appstoreToken); // appstore token
debug('Setup complete'); log('Setup complete');
} }
async function cleanup() { async function cleanup() {
debug('Cleaning up'); log('Cleaning up');
await server.stop(); await server.stop();
await oidcServer.stop(); await oidcServer.stop();
if (!nock.isActive()) nock.activate(); if (!nock.isActive()) nock.activate();
debug('Cleaned up'); log('Cleaned up');
} }
function clearMailQueue() { function clearMailQueue() {
@@ -187,7 +187,7 @@ async function checkMails(number) {
} }
async function waitForTask(taskId) { async function waitForTask(taskId) {
debug(`Waiting for task: ${taskId}`); log(`Waiting for task: ${taskId}`);
for (let i = 0; i < 30; i++) { for (let i = 0; i < 30; i++) {
const result = await tasks.get(taskId); const result = await tasks.get(taskId);
@@ -197,7 +197,7 @@ async function waitForTask(taskId) {
throw new Error(`Task ${taskId} failed: ${result.error.message} - ${result.error.stack}`); throw new Error(`Task ${taskId} failed: ${result.error.message} - ${result.error.stack}`);
} }
await timers.setTimeout(2000); await timers.setTimeout(2000);
debug(`Waiting for task to ${taskId} finish`); log(`Waiting for task to ${taskId} finish`);
} }
throw new Error(`Task ${taskId} never finished`); throw new Error(`Task ${taskId} never finished`);
} }
@@ -206,16 +206,16 @@ async function waitForAsyncTask(es) {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
const messages = []; const messages = [];
es.addEventListener('message', function (message) { es.addEventListener('message', function (message) {
debug(`waitForAsyncTask: ${message.data}`); log(`waitForAsyncTask: ${message.data}`);
messages.push(JSON.parse(message.data)); messages.push(JSON.parse(message.data));
if (messages[messages.length-1].type === 'done') { if (messages[messages.length-1].type === 'done') {
debug('waitForAsyncTask: finished'); log('waitForAsyncTask: finished');
es.close(); es.close();
resolve(messages); resolve(messages);
} }
}); });
es.addEventListener('error', function (error) { es.addEventListener('error', function (error) {
debug('waitForAsyncTask: errored', error); log('waitForAsyncTask: errored', error);
es.close(); es.close();
const e = new Error(error.message); const e = new Error(error.message);
e.code = error.code; e.code = error.code;

32
src/scheduler.js
View File

@@ -4,12 +4,12 @@ import BoxError from './boxerror.js';
import cloudron from './cloudron.js'; import cloudron from './cloudron.js';
import constants from './constants.js'; import constants from './constants.js';
import { CronJob } from 'cron'; import { CronJob } from 'cron';
import debugModule from 'debug'; import logger from './logger.js';
import docker from './docker.js'; import docker from './docker.js';
import safe from 'safetydance'; import safe from 'safetydance';
import _ from './underscore.js'; import _ from './underscore.js';
const debug = debugModule('box:scheduler'); const { log, trace } = logger('scheduler');
const gState = {}; // appId -> { containerId, schedulerConfig (manifest+crontab), cronjobs } const gState = {}; // appId -> { containerId, schedulerConfig (manifest+crontab), cronjobs }
@@ -17,12 +17,12 @@ const gSuspendedAppIds = new Set(); // suspended because some apptask is running
// TODO: this should probably also stop existing jobs to completely prevent race but the code is not re-entrant // TODO: this should probably also stop existing jobs to completely prevent race but the code is not re-entrant
function suspendAppJobs(appId) { function suspendAppJobs(appId) {
debug(`suspendAppJobs: ${appId}`); log(`suspendAppJobs: ${appId}`);
gSuspendedAppIds.add(appId); gSuspendedAppIds.add(appId);
} }
function resumeAppJobs(appId) { function resumeAppJobs(appId) {
debug(`resumeAppJobs: ${appId}`); log(`resumeAppJobs: ${appId}`);
gSuspendedAppIds.delete(appId); gSuspendedAppIds.delete(appId);
} }
@@ -44,7 +44,7 @@ async function runTask(appId, taskName) {
if (!error && data?.State?.Running === true) { if (!error && data?.State?.Running === true) {
const jobStartTime = new Date(data.State.StartedAt); // iso 8601 const jobStartTime = new Date(data.State.StartedAt); // iso 8601
if ((new Date() - jobStartTime) < JOB_MAX_TIME) return; if ((new Date() - jobStartTime) < JOB_MAX_TIME) return;
debug(`runTask: ${containerName} is running too long, restarting`); log(`runTask: ${containerName} is running too long, restarting`);
} }
await docker.restartContainer(containerName); await docker.restartContainer(containerName);
@@ -65,10 +65,10 @@ async function createJobs(app, schedulerConfig) {
// stopJobs only deletes jobs since previous sync. This means that when box code restarts, none of the containers // stopJobs only deletes jobs since previous sync. This means that when box code restarts, none of the containers
// are removed. The deleteContainer here ensures we re-create the cron containers with the latest config // are removed. The deleteContainer here ensures we re-create the cron containers with the latest config
await safe(docker.deleteContainer(containerName)); // ignore error await safe(docker.deleteContainer(containerName)); // ignore error
const [error] = await safe(docker.createSubcontainer(app, containerName, [ '/bin/sh', '-c', command ], {} /* options */), { debug }); const [error] = await safe(docker.createSubcontainer(app, containerName, [ '/bin/sh', '-c', command ], {} /* options */), { debug: log });
if (error && error.reason !== BoxError.ALREADY_EXISTS) continue; if (error && error.reason !== BoxError.ALREADY_EXISTS) continue;
debug(`createJobs: ${taskName} (${app.fqdn}) will run in container ${containerName}`); log(`createJobs: ${taskName} (${app.fqdn}) will run in container ${containerName}`);
let cronTime; let cronTime;
if (schedule === '@service') { if (schedule === '@service') {
@@ -82,7 +82,7 @@ async function createJobs(app, schedulerConfig) {
cronTime, cronTime,
onTick: async () => { onTick: async () => {
const [taskError] = await safe(runTask(appId, taskName)); // put the app id in closure, so we don't use the outdated app object by mistake const [taskError] = await safe(runTask(appId, taskName)); // put the app id in closure, so we don't use the outdated app object by mistake
if (taskError) debug(`could not run task ${taskName} : ${taskError.message}`); if (taskError) log(`could not run task ${taskName} : ${taskError.message}`);
}, },
start: true, start: true,
timeZone: tz timeZone: tz
@@ -105,15 +105,15 @@ async function deleteAppJobs(appId, appState) {
const containerName = `${appId}-${taskName}`; const containerName = `${appId}-${taskName}`;
const [error] = await safe(docker.deleteContainer(containerName)); const [error] = await safe(docker.deleteContainer(containerName));
if (error) debug(`deleteAppJobs: failed to delete task container with name ${containerName} : ${error.message}`); if (error) log(`deleteAppJobs: failed to delete task container with name ${containerName} : ${error.message}`);
} }
} }
async function deleteJobs() { async function deleteJobs() {
for (const appId of Object.keys(gState)) { for (const appId of Object.keys(gState)) {
debug(`deleteJobs: removing jobs of ${appId}`); log(`deleteJobs: removing jobs of ${appId}`);
const [error] = await safe(deleteAppJobs(appId, gState[appId])); const [error] = await safe(deleteAppJobs(appId, gState[appId]));
if (error) debug(`deleteJobs: error stopping jobs of removed app ${appId}: ${error.message}`); if (error) log(`deleteJobs: error stopping jobs of removed app ${appId}: ${error.message}`);
delete gState[appId]; delete gState[appId];
} }
} }
@@ -125,12 +125,12 @@ async function sync() {
const allAppIds = allApps.map(app => app.id); const allAppIds = allApps.map(app => app.id);
const removedAppIds = _.difference(Object.keys(gState), allAppIds); const removedAppIds = _.difference(Object.keys(gState), allAppIds);
if (removedAppIds.length !== 0) debug(`sync: stopping jobs of removed apps ${JSON.stringify(removedAppIds)}`); if (removedAppIds.length !== 0) log(`sync: stopping jobs of removed apps ${JSON.stringify(removedAppIds)}`);
for (const appId of removedAppIds) { for (const appId of removedAppIds) {
debug(`sync: removing jobs of ${appId}`); log(`sync: removing jobs of ${appId}`);
const [error] = await safe(deleteAppJobs(appId, gState[appId])); const [error] = await safe(deleteAppJobs(appId, gState[appId]));
if (error) debug(`sync: error stopping jobs of removed app ${appId}: ${error.message}`); if (error) log(`sync: error stopping jobs of removed app ${appId}: ${error.message}`);
delete gState[appId]; delete gState[appId];
} }
@@ -143,10 +143,10 @@ async function sync() {
if (_.isEqual(appState.schedulerConfig, schedulerConfig) && appState.containerId === app.containerId) continue; // nothing changed if (_.isEqual(appState.schedulerConfig, schedulerConfig) && appState.containerId === app.containerId) continue; // nothing changed
} }
debug(`sync: clearing jobs of ${app.id} (${app.fqdn})`); log(`sync: clearing jobs of ${app.id} (${app.fqdn})`);
const [error] = await safe(deleteAppJobs(app.id, appState)); const [error] = await safe(deleteAppJobs(app.id, appState));
if (error) debug(`sync: error stopping jobs of ${app.id} : ${error.message}`); if (error) log(`sync: error stopping jobs of ${app.id} : ${error.message}`);
if (!schedulerConfig) { // updated app version removed scheduler addon if (!schedulerConfig) { // updated app version removed scheduler addon
delete gState[app.id]; delete gState[app.id];

10
src/scripts/backupupload.js
View File

@@ -2,10 +2,10 @@
import backuptask from '../backuptask.js'; import backuptask from '../backuptask.js';
import database from '../database.js'; import database from '../database.js';
import debugModule from 'debug'; import logger from '../logger.js';
import safe from 'safetydance'; import safe from 'safetydance';
const debug = debugModule('box:backupupload'); const { log, trace } = logger('backupupload');
// --check is used by run-tests to verify sudo access works. Caller must set BOX_ENV (e.g. BOX_ENV=test). // --check is used by run-tests to verify sudo access works. Caller must set BOX_ENV (e.g. BOX_ENV=test).
if (process.argv[2] === '--check') { if (process.argv[2] === '--check') {
@@ -18,7 +18,7 @@ const remotePath = process.argv[2];
const format = process.argv[3]; const format = process.argv[3];
const dataLayoutString = process.argv[4]; const dataLayoutString = process.argv[4];
debug(`Backing up ${dataLayoutString} to ${remotePath}`); log(`Backing up ${dataLayoutString} to ${remotePath}`);
process.on('SIGTERM', function () { process.on('SIGTERM', function () {
process.exit(0); process.exit(0);
@@ -26,7 +26,7 @@ process.on('SIGTERM', function () {
// this can happen when the backup task is terminated (not box code) // this can happen when the backup task is terminated (not box code)
process.on('disconnect', function () { process.on('disconnect', function () {
debug('parent process died'); log('parent process died');
process.exit(0); process.exit(0);
}); });
@@ -45,7 +45,7 @@ function throttledProgressCallback(msecs) {
await database.initialize(); await database.initialize();
const [uploadError, result] = await safe(backuptask.upload(remotePath, format, dataLayoutString, throttledProgressCallback(5000))); const [uploadError, result] = await safe(backuptask.upload(remotePath, format, dataLayoutString, throttledProgressCallback(5000)));
debug('upload completed. error: %o', uploadError); log('upload completed. error: %o', uploadError);
process.send({ result, errorMessage: uploadError?.message }); process.send({ result, errorMessage: uploadError?.message });

2
src/scripts/starttask.sh
View File

@@ -45,7 +45,7 @@ options="-p TimeoutStopSec=10s -p MemoryMax=${memory_limit_mb}M -p OOMScoreAdjus
# it seems systemd-run does not return the exit status of the process despite --wait but atleast it waits # it seems systemd-run does not return the exit status of the process despite --wait but atleast it waits
if ! systemd-run --unit "${service_name}" --wait --uid=${id} --gid=${id} \ if ! systemd-run --unit "${service_name}" --wait --uid=${id} --gid=${id} \
-p TimeoutStopSec=2s -p MemoryMax=${memory_limit_mb}M -p OOMScoreAdjust=${oom_score_adjust} --nice "${nice}" \ -p TimeoutStopSec=2s -p MemoryMax=${memory_limit_mb}M -p OOMScoreAdjust=${oom_score_adjust} --nice "${nice}" \
--setenv HOME=${HOME} --setenv USER=${SUDO_USER} --setenv DEBUG=box:* --setenv BOX_ENV=${BOX_ENV} --setenv NODE_ENV=production \ --setenv HOME=${HOME} --setenv USER=${SUDO_USER} --setenv BOX_ENV=${BOX_ENV} --setenv NODE_ENV=production \
"${task_worker}" "${task_id}" "${logfile}"; then "${task_worker}" "${task_id}" "${logfile}"; then
echo "Service ${service_name} failed to run" # this only happens if the path to task worker itself is wrong echo "Service ${service_name} failed to run" # this only happens if the path to task worker itself is wrong
fi fi

12
src/server.js
View File

@@ -1,7 +1,7 @@
import assert from 'node:assert'; import assert from 'node:assert';
import AuditSource from './auditsource.js'; import AuditSource from './auditsource.js';
import constants from './constants.js'; import constants from './constants.js';
import debugModule from 'debug'; import logger from './logger.js';
import eventlog from './eventlog.js'; import eventlog from './eventlog.js';
import express from 'express'; import express from 'express';
import http from 'node:http'; import http from 'node:http';
@@ -14,14 +14,14 @@ import users from './users.js';
import util from 'node:util'; import util from 'node:util';
import { WebSocketServer } from 'ws'; import { WebSocketServer } from 'ws';
const debug = debugModule('box:server'); const { log, trace } = logger('server');
let gHttpServer = null; let gHttpServer = null;
function notFoundHandler(req, res, next) { function notFoundHandler(req, res, next) {
const cleanUrl = req.url.replace(/(access_token=)[^&]+/, '$1' + '<redacted>'); const cleanUrl = req.url.replace(/(access_token=)[^&]+/, '$1' + '<redacted>');
debug(`no such route: ${req.method} ${cleanUrl}`); log(`no such route: ${req.method} ${cleanUrl}`);
return next(new HttpError(404, 'No such route')); return next(new HttpError(404, 'No such route'));
} }
@@ -507,9 +507,9 @@ async function initializeExpressSync() {
async function start() { async function start() {
assert(gHttpServer === null, 'Server is already up and running.'); assert(gHttpServer === null, 'Server is already up and running.');
debug('=========================================='); log('==========================================');
debug(` Cloudron ${constants.VERSION} `); log(` Cloudron ${constants.VERSION} `);
debug('=========================================='); log('==========================================');
await platform.initialize(); await platform.initialize();

210
src/services.js
View File

@@ -7,7 +7,7 @@ import branding from './branding.js';
import constants from './constants.js'; import constants from './constants.js';
import crypto from 'node:crypto'; import crypto from 'node:crypto';
import dashboard from './dashboard.js'; import dashboard from './dashboard.js';
import debugModule from 'debug'; import logger from './logger.js';
import dig from './dig.js'; import dig from './dig.js';
import docker from './docker.js'; import docker from './docker.js';
import eventlog from './eventlog.js'; import eventlog from './eventlog.js';
@@ -31,7 +31,7 @@ import sftp from './sftp.js';
import shellModule from './shell.js'; import shellModule from './shell.js';
import superagent from '@cloudron/superagent'; import superagent from '@cloudron/superagent';
const debug = debugModule('box:services'); const { log, trace } = logger('services');
const shell = shellModule('services'); const shell = shellModule('services');
const SERVICE_STATUS_STARTING = 'starting'; const SERVICE_STATUS_STARTING = 'starting';
@@ -169,7 +169,7 @@ async function startAppServices(app) {
const [error] = await safe(APP_SERVICES[addon].start(instance)); // assume addons name is service name const [error] = await safe(APP_SERVICES[addon].start(instance)); // assume addons name is service name
// error ignored because we don't want "start app" to error. use can fix it from Services // error ignored because we don't want "start app" to error. use can fix it from Services
if (error) debug(`startAppServices: ${addon}:${instance}. %o`, error); if (error) log(`startAppServices: ${addon}:${instance}. %o`, error);
} }
} }
@@ -182,7 +182,7 @@ async function stopAppServices(app) {
const [error] = await safe(APP_SERVICES[addon].stop(instance)); // assume addons name is service name const [error] = await safe(APP_SERVICES[addon].stop(instance)); // assume addons name is service name
// error ignored because we don't want "start app" to error. use can fix it from Services // error ignored because we don't want "start app" to error. use can fix it from Services
if (error) debug(`stopAppServices: ${addon}:${instance}. %o`, error); if (error) log(`stopAppServices: ${addon}:${instance}. %o`, error);
} }
} }
@@ -190,11 +190,11 @@ async function waitForContainer(containerName, tokenEnvName) {
assert.strictEqual(typeof containerName, 'string'); assert.strictEqual(typeof containerName, 'string');
assert.strictEqual(typeof tokenEnvName, 'string'); assert.strictEqual(typeof tokenEnvName, 'string');
debug(`Waiting for ${containerName}`); log(`Waiting for ${containerName}`);
const result = await getContainerDetails(containerName, tokenEnvName); const result = await getContainerDetails(containerName, tokenEnvName);
await promiseRetry({ times: 20, interval: 15000, debug }, async () => { await promiseRetry({ times: 20, interval: 15000, debug: log }, async () => {
const [networkError, response] = await safe(superagent.get(`http://${result.ip}:3000/healthcheck?access_token=${result.token}`) const [networkError, response] = await safe(superagent.get(`http://${result.ip}:3000/healthcheck?access_token=${result.token}`)
.timeout(20000) .timeout(20000)
.ok(() => true)); .ok(() => true));
@@ -217,7 +217,7 @@ async function ensureServiceRunning(serviceName) {
if (error) throw new BoxError(BoxError.ADDONS_ERROR, `${serviceName} container not found`); if (error) throw new BoxError(BoxError.ADDONS_ERROR, `${serviceName} container not found`);
if (container.State?.Running) return; if (container.State?.Running) return;
debug(`ensureServiceRunning: starting ${serviceName}`); log(`ensureServiceRunning: starting ${serviceName}`);
await docker.startContainer(serviceName); await docker.startContainer(serviceName);
if (tokenEnvNames[serviceName]) await waitForContainer(serviceName, tokenEnvNames[serviceName]); if (tokenEnvNames[serviceName]) await waitForContainer(serviceName, tokenEnvNames[serviceName]);
@@ -232,22 +232,22 @@ async function stopUnusedServices() {
} }
} }
debug(`stopUnusedServices: used addons - ${[...usedAddons]}`); log(`stopUnusedServices: used addons - ${[...usedAddons]}`);
for (const name of LAZY_SERVICES) { for (const name of LAZY_SERVICES) {
if (usedAddons.has(name)) continue; if (usedAddons.has(name)) continue;
debug(`stopUnusedServices: stopping ${name} (no apps use it)`); log(`stopUnusedServices: stopping ${name} (no apps use it)`);
await safe(docker.stopContainer(name), { debug }); await safe(docker.stopContainer(name), { debug: log });
} }
} }
async function exportDatabase(addon) { async function exportDatabase(addon) {
assert.strictEqual(typeof addon, 'string'); assert.strictEqual(typeof addon, 'string');
debug(`exportDatabase: exporting ${addon}`); log(`exportDatabase: exporting ${addon}`);
if (fs.existsSync(path.join(paths.ADDON_CONFIG_DIR, `exported-${addon}`))) { if (fs.existsSync(path.join(paths.ADDON_CONFIG_DIR, `exported-${addon}`))) {
debug(`exportDatabase: already exported addon ${addon} in previous run`); log(`exportDatabase: already exported addon ${addon} in previous run`);
return; return;
} }
@@ -257,12 +257,12 @@ async function exportDatabase(addon) {
if (!app.manifest.addons || !(addon in app.manifest.addons)) continue; // app doesn't use the addon if (!app.manifest.addons || !(addon in app.manifest.addons)) continue; // app doesn't use the addon
if (app.installationState === apps.ISTATE_ERROR) continue; // missing db causes crash in old app addon containers if (app.installationState === apps.ISTATE_ERROR) continue; // missing db causes crash in old app addon containers
debug(`exportDatabase: exporting addon ${addon} of app ${app.id}`); log(`exportDatabase: exporting addon ${addon} of app ${app.id}`);
// eslint-disable-next-line no-use-before-define -- circular: ADDONS references setup fns, setup fns call exportDatabase // eslint-disable-next-line no-use-before-define -- circular: ADDONS references setup fns, setup fns call exportDatabase
const [error] = await safe(ADDONS[addon].backup(app, app.manifest.addons[addon])); const [error] = await safe(ADDONS[addon].backup(app, app.manifest.addons[addon]));
if (error) { if (error) {
debug(`exportDatabase: error exporting ${addon} of app ${app.id}. %o`, error); log(`exportDatabase: error exporting ${addon} of app ${app.id}. %o`, error);
// for errored apps, we can ignore if export had an error // for errored apps, we can ignore if export had an error
if (app.installationState === apps.ISTATE_ERROR) continue; if (app.installationState === apps.ISTATE_ERROR) continue;
throw error; throw error;
@@ -279,19 +279,19 @@ async function exportDatabase(addon) {
async function importDatabase(addon) { async function importDatabase(addon) {
assert.strictEqual(typeof addon, 'string'); assert.strictEqual(typeof addon, 'string');
debug(`importDatabase: importing ${addon}`); log(`importDatabase: importing ${addon}`);
const allApps = await apps.list(); const allApps = await apps.list();
for (const app of allApps) { for (const app of allApps) {
if (!app.manifest.addons || !(addon in app.manifest.addons)) continue; // app doesn't use the addon if (!app.manifest.addons || !(addon in app.manifest.addons)) continue; // app doesn't use the addon
debug(`importDatabase: importing addon ${addon} of app ${app.id}`); log(`importDatabase: importing addon ${addon} of app ${app.id}`);
const [error] = await safe(importAppDatabase(app, addon)); // eslint-disable-line no-use-before-define const [error] = await safe(importAppDatabase(app, addon)); // eslint-disable-line no-use-before-define
if (!error) continue; if (!error) continue;
debug(`importDatabase: error importing ${addon} of app ${app.id}. Marking as errored. %o`, error); log(`importDatabase: error importing ${addon} of app ${app.id}. Marking as errored. %o`, error);
// FIXME: there is no way to 'repair' if we are here. we need to make a separate apptask that re-imports db // FIXME: there is no way to 'repair' if we are here. we need to make a separate apptask that re-imports db
// not clear, if repair workflow should be part of addon or per-app // not clear, if repair workflow should be part of addon or per-app
await safe(apps.update(app.id, { installationState: apps.ISTATE_ERROR, error: { message: error.message } })); await safe(apps.update(app.id, { installationState: apps.ISTATE_ERROR, error: { message: error.message } }));
@@ -304,7 +304,7 @@ async function setupLocalStorage(app, options) {
assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof app, 'object');
assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof options, 'object');
debug('setupLocalStorage'); log('setupLocalStorage');
const volumeDataDir = await apps.getStorageDir(app); const volumeDataDir = await apps.getStorageDir(app);
@@ -316,7 +316,7 @@ async function clearLocalStorage(app, options) {
assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof app, 'object');
assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof options, 'object');
debug('clearLocalStorage'); log('clearLocalStorage');
const volumeDataDir = await apps.getStorageDir(app); const volumeDataDir = await apps.getStorageDir(app);
const [error] = await safe(shell.sudo([ CLEARVOLUME_CMD, volumeDataDir ], {})); const [error] = await safe(shell.sudo([ CLEARVOLUME_CMD, volumeDataDir ], {}));
@@ -327,7 +327,7 @@ async function teardownLocalStorage(app, options) {
assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof app, 'object');
assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof options, 'object');
debug('teardownLocalStorage'); log('teardownLocalStorage');
const volumeDataDir = await apps.getStorageDir(app); const volumeDataDir = await apps.getStorageDir(app);
const [error] = await safe(shell.sudo([ RMVOLUME_CMD, volumeDataDir ], {})); const [error] = await safe(shell.sudo([ RMVOLUME_CMD, volumeDataDir ], {}));
@@ -340,7 +340,7 @@ async function backupSqlite(app, options) {
assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof app, 'object');
assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof options, 'object');
debug('Backing up sqlite'); log('Backing up sqlite');
const volumeDataDir = await apps.getStorageDir(app); const volumeDataDir = await apps.getStorageDir(app);
@@ -372,7 +372,7 @@ async function restoreSqlite(app, options) {
assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof app, 'object');
assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof options, 'object');
debug('Restoring sqlite'); log('Restoring sqlite');
const volumeDataDir = await apps.getStorageDir(app); const volumeDataDir = await apps.getStorageDir(app);
@@ -408,7 +408,7 @@ async function setupPersistentDirs(app) {
if (!app.manifest.persistentDirs) return; if (!app.manifest.persistentDirs) return;
debug('setupPersistentDirs'); log('setupPersistentDirs');
for (const dir of app.manifest.persistentDirs) { for (const dir of app.manifest.persistentDirs) {
const hostDir = persistentDirHostPath(app.id, dir); const hostDir = persistentDirHostPath(app.id, dir);
@@ -422,7 +422,7 @@ async function teardownPersistentDirs(app) {
if (!app.manifest.persistentDirs) return; if (!app.manifest.persistentDirs) return;
debug('teardownPersistentDirs'); log('teardownPersistentDirs');
for (const dir of app.manifest.persistentDirs) { for (const dir of app.manifest.persistentDirs) {
const hostDir = persistentDirHostPath(app.id, dir); const hostDir = persistentDirHostPath(app.id, dir);
@@ -438,7 +438,7 @@ async function runBackupCommand(app) {
if (!app.manifest.backupCommand) return; if (!app.manifest.backupCommand) return;
debug('runBackupCommand'); log('runBackupCommand');
const volumeDataDir = await apps.getStorageDir(app); const volumeDataDir = await apps.getStorageDir(app);
const dataDirMount = volumeDataDir ? `-v ${volumeDataDir}:/app/data` : ''; const dataDirMount = volumeDataDir ? `-v ${volumeDataDir}:/app/data` : '';
@@ -462,7 +462,7 @@ async function runRestoreCommand(app) {
if (!app.manifest.restoreCommand) return; if (!app.manifest.restoreCommand) return;
debug('runRestoreCommand'); log('runRestoreCommand');
const volumeDataDir = await apps.getStorageDir(app); const volumeDataDir = await apps.getStorageDir(app);
const dataDirMount = volumeDataDir ? `-v ${volumeDataDir}:/app/data` : ''; const dataDirMount = volumeDataDir ? `-v ${volumeDataDir}:/app/data` : '';
@@ -504,7 +504,7 @@ async function setupTurn(app, options) {
{ name: 'CLOUDRON_TURN_SECRET', value: turnSecret } { name: 'CLOUDRON_TURN_SECRET', value: turnSecret }
]; ];
debug('Setting up TURN'); log('Setting up TURN');
await addonConfigs.set(app.id, 'turn', env); await addonConfigs.set(app.id, 'turn', env);
} }
@@ -513,7 +513,7 @@ async function teardownTurn(app, options) {
assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof app, 'object');
assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof options, 'object');
debug('Tearing down TURN'); log('Tearing down TURN');
await addonConfigs.unset(app.id, 'turn'); await addonConfigs.unset(app.id, 'turn');
} }
@@ -543,7 +543,7 @@ async function setupEmail(app, options) {
{ name: 'CLOUDRON_EMAIL_LDAP_MAILBOXES_BASE_DN', value: 'ou=mailboxes,dc=cloudron' } { name: 'CLOUDRON_EMAIL_LDAP_MAILBOXES_BASE_DN', value: 'ou=mailboxes,dc=cloudron' }
]; ];
debug('Setting up Email'); log('Setting up Email');
await addonConfigs.set(app.id, 'email', env); await addonConfigs.set(app.id, 'email', env);
} }
@@ -552,7 +552,7 @@ async function teardownEmail(app, options) {
assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof app, 'object');
assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof options, 'object');
debug('Tearing down Email'); log('Tearing down Email');
await addonConfigs.unset(app.id, 'email'); await addonConfigs.unset(app.id, 'email');
} }
@@ -574,7 +574,7 @@ async function setupLdap(app, options) {
{ name: 'CLOUDRON_LDAP_BIND_PASSWORD', value: hat(4 * 128) } // this is ignored { name: 'CLOUDRON_LDAP_BIND_PASSWORD', value: hat(4 * 128) } // this is ignored
]; ];
debug('Setting up LDAP'); log('Setting up LDAP');
await addonConfigs.set(app.id, 'ldap', env); await addonConfigs.set(app.id, 'ldap', env);
} }
@@ -583,7 +583,7 @@ async function teardownLdap(app, options) {
assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof app, 'object');
assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof options, 'object');
debug('Tearing down LDAP'); log('Tearing down LDAP');
await addonConfigs.unset(app.id, 'ldap'); await addonConfigs.unset(app.id, 'ldap');
} }
@@ -592,7 +592,7 @@ async function setupSendMail(app, options) {
assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof app, 'object');
assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof options, 'object');
debug('Setting up SendMail'); log('Setting up SendMail');
const disabled = app.manifest.addons.sendmail.optional && !app.enableMailbox; const disabled = app.manifest.addons.sendmail.optional && !app.enableMailbox;
if (disabled) return await addonConfigs.unset(app.id, 'sendmail'); if (disabled) return await addonConfigs.unset(app.id, 'sendmail');
@@ -616,7 +616,7 @@ async function setupSendMail(app, options) {
if (app.manifest.addons.sendmail.supportsDisplayName) env.push({ name: 'CLOUDRON_MAIL_FROM_DISPLAY_NAME', value: app.mailboxDisplayName }); if (app.manifest.addons.sendmail.supportsDisplayName) env.push({ name: 'CLOUDRON_MAIL_FROM_DISPLAY_NAME', value: app.mailboxDisplayName });
debug('Setting sendmail addon config to %j', env); log('Setting sendmail addon config to %j', env);
await addonConfigs.set(app.id, 'sendmail', env); await addonConfigs.set(app.id, 'sendmail', env);
} }
@@ -624,7 +624,7 @@ async function teardownSendMail(app, options) {
assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof app, 'object');
assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof options, 'object');
debug('Tearing down sendmail'); log('Tearing down sendmail');
await addonConfigs.unset(app.id, 'sendmail'); await addonConfigs.unset(app.id, 'sendmail');
} }
@@ -633,7 +633,7 @@ async function setupRecvMail(app, options) {
assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof app, 'object');
assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof options, 'object');
debug('setupRecvMail: setting up recvmail'); log('setupRecvMail: setting up recvmail');
if (!app.enableInbox) return await addonConfigs.unset(app.id, 'recvmail'); if (!app.enableInbox) return await addonConfigs.unset(app.id, 'recvmail');
@@ -653,7 +653,7 @@ async function setupRecvMail(app, options) {
{ name: 'CLOUDRON_MAIL_TO_DOMAIN', value: app.inboxDomain }, { name: 'CLOUDRON_MAIL_TO_DOMAIN', value: app.inboxDomain },
]; ];
debug('setupRecvMail: setting recvmail addon config to %j', env); log('setupRecvMail: setting recvmail addon config to %j', env);
await addonConfigs.set(app.id, 'recvmail', env); await addonConfigs.set(app.id, 'recvmail', env);
} }
@@ -661,7 +661,7 @@ async function teardownRecvMail(app, options) {
assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof app, 'object');
assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof options, 'object');
debug('teardownRecvMail: tearing down recvmail'); log('teardownRecvMail: tearing down recvmail');
await addonConfigs.unset(app.id, 'recvmail'); await addonConfigs.unset(app.id, 'recvmail');
} }
@@ -685,7 +685,7 @@ async function startMysql(existingInfra) {
const upgrading = existingInfra.version !== 'none' && requiresUpgrade(existingInfra.images.mysql, image); const upgrading = existingInfra.version !== 'none' && requiresUpgrade(existingInfra.images.mysql, image);
if (upgrading) { if (upgrading) {
debug('startMysql: mysql will be upgraded'); log('startMysql: mysql will be upgraded');
await exportDatabase('mysql'); await exportDatabase('mysql');
} }
@@ -711,11 +711,11 @@ async function startMysql(existingInfra) {
--cap-add SYS_NICE \ --cap-add SYS_NICE \
${readOnly} -v /tmp -v /run ${image} ${cmd}`; ${readOnly} -v /tmp -v /run ${image} ${cmd}`;
debug('startMysql: stopping and deleting previous mysql container'); log('startMysql: stopping and deleting previous mysql container');
await docker.stopContainer('mysql'); await docker.stopContainer('mysql');
await docker.deleteContainer('mysql'); await docker.deleteContainer('mysql');
debug('startMysql: starting mysql container'); log('startMysql: starting mysql container');
await shell.bash(runCmd, { encoding: 'utf8' }); await shell.bash(runCmd, { encoding: 'utf8' });
if (!serviceConfig.recoveryMode) { if (!serviceConfig.recoveryMode) {
@@ -730,7 +730,7 @@ async function setupMySql(app, options) {
assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof app, 'object');
assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof options, 'object');
debug('Setting up mysql'); log('Setting up mysql');
await ensureServiceRunning('mysql'); await ensureServiceRunning('mysql');
@@ -770,7 +770,7 @@ async function setupMySql(app, options) {
); );
} }
debug('Setting mysql addon config to %j', env); log('Setting mysql addon config to %j', env);
await addonConfigs.set(app.id, 'mysql', env); await addonConfigs.set(app.id, 'mysql', env);
} }
@@ -813,7 +813,7 @@ async function backupMySql(app, options) {
const database = mysqlDatabaseName(app.id); const database = mysqlDatabaseName(app.id);
debug('Backing up mysql'); log('Backing up mysql');
const result = await getContainerDetails('mysql', 'CLOUDRON_MYSQL_TOKEN'); const result = await getContainerDetails('mysql', 'CLOUDRON_MYSQL_TOKEN');
@@ -829,7 +829,7 @@ async function restoreMySql(app, options) {
const database = mysqlDatabaseName(app.id); const database = mysqlDatabaseName(app.id);
debug('restoreMySql'); log('restoreMySql');
const result = await getContainerDetails('mysql', 'CLOUDRON_MYSQL_TOKEN'); const result = await getContainerDetails('mysql', 'CLOUDRON_MYSQL_TOKEN');
@@ -855,7 +855,7 @@ async function startPostgresql(existingInfra) {
const upgrading = existingInfra.version !== 'none' && requiresUpgrade(existingInfra.images.postgresql, image); const upgrading = existingInfra.version !== 'none' && requiresUpgrade(existingInfra.images.postgresql, image);
if (upgrading) { if (upgrading) {
debug('startPostgresql: postgresql will be upgraded'); log('startPostgresql: postgresql will be upgraded');
await exportDatabase('postgresql'); await exportDatabase('postgresql');
} }
@@ -880,11 +880,11 @@ async function startPostgresql(existingInfra) {
--label isCloudronManaged=true \ --label isCloudronManaged=true \
${readOnly} -v /tmp -v /run ${image} ${cmd}`; ${readOnly} -v /tmp -v /run ${image} ${cmd}`;
debug('startPostgresql: stopping and deleting previous postgresql container'); log('startPostgresql: stopping and deleting previous postgresql container');
await docker.stopContainer('postgresql'); await docker.stopContainer('postgresql');
await docker.deleteContainer('postgresql'); await docker.deleteContainer('postgresql');
debug('startPostgresql: starting postgresql container'); log('startPostgresql: starting postgresql container');
await shell.bash(runCmd, { encoding: 'utf8' }); await shell.bash(runCmd, { encoding: 'utf8' });
if (!serviceConfig.recoveryMode) { if (!serviceConfig.recoveryMode) {
@@ -899,7 +899,7 @@ async function setupPostgreSql(app, options) {
assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof app, 'object');
assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof options, 'object');
debug('Setting up postgresql'); log('Setting up postgresql');
await ensureServiceRunning('postgresql'); await ensureServiceRunning('postgresql');
@@ -931,7 +931,7 @@ async function setupPostgreSql(app, options) {
{ name: 'CLOUDRON_POSTGRESQL_DATABASE', value: data.database } { name: 'CLOUDRON_POSTGRESQL_DATABASE', value: data.database }
]; ];
debug('Setting postgresql addon config to %j', env); log('Setting postgresql addon config to %j', env);
await addonConfigs.set(app.id, 'postgresql', env); await addonConfigs.set(app.id, 'postgresql', env);
} }
@@ -942,7 +942,7 @@ async function clearPostgreSql(app, options) {
const { database, username } = postgreSqlNames(app.id); const { database, username } = postgreSqlNames(app.id);
const locale = options.locale || 'C'; const locale = options.locale || 'C';
debug('Clearing postgresql'); log('Clearing postgresql');
const result = await getContainerDetails('postgresql', 'CLOUDRON_POSTGRESQL_TOKEN'); const result = await getContainerDetails('postgresql', 'CLOUDRON_POSTGRESQL_TOKEN');
@@ -972,7 +972,7 @@ async function backupPostgreSql(app, options) {
assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof app, 'object');
assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof options, 'object');
debug('Backing up postgresql'); log('Backing up postgresql');
const { database } = postgreSqlNames(app.id); const { database } = postgreSqlNames(app.id);
@@ -986,7 +986,7 @@ async function restorePostgreSql(app, options) {
assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof app, 'object');
assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof options, 'object');
debug('Restore postgresql'); log('Restore postgresql');
const { database, username } = postgreSqlNames(app.id); const { database, username } = postgreSqlNames(app.id);
@@ -1008,7 +1008,7 @@ async function startMongodb(existingInfra) {
const upgrading = existingInfra.version !== 'none' && requiresUpgrade(existingInfra.images.mongodb, image); const upgrading = existingInfra.version !== 'none' && requiresUpgrade(existingInfra.images.mongodb, image);
if (upgrading) { if (upgrading) {
debug('startMongodb: mongodb will be upgraded'); log('startMongodb: mongodb will be upgraded');
await exportDatabase('mongodb'); await exportDatabase('mongodb');
} }
@@ -1032,16 +1032,16 @@ async function startMongodb(existingInfra) {
--label isCloudronManaged=true \ --label isCloudronManaged=true \
${readOnly} -v /tmp -v /run ${image} ${cmd}`; ${readOnly} -v /tmp -v /run ${image} ${cmd}`;
debug('startMongodb: stopping and deleting previous mongodb container'); log('startMongodb: stopping and deleting previous mongodb container');
await docker.stopContainer('mongodb'); await docker.stopContainer('mongodb');
await docker.deleteContainer('mongodb'); await docker.deleteContainer('mongodb');
if (!await hasAVX()) { if (!await hasAVX()) {
debug('startMongodb: not starting mongodb because CPU does not have AVX'); log('startMongodb: not starting mongodb because CPU does not have AVX');
return; return;
} }
debug('startMongodb: starting mongodb container'); log('startMongodb: starting mongodb container');
await shell.bash(runCmd, { encoding: 'utf8' }); await shell.bash(runCmd, { encoding: 'utf8' });
if (!serviceConfig.recoveryMode) { if (!serviceConfig.recoveryMode) {
@@ -1056,7 +1056,7 @@ async function setupMongoDb(app, options) {
assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof app, 'object');
assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof options, 'object');
debug('Setting up mongodb'); log('Setting up mongodb');
await ensureServiceRunning('mongodb'); await ensureServiceRunning('mongodb');
@@ -1095,7 +1095,7 @@ async function setupMongoDb(app, options) {
env.push({ name: 'CLOUDRON_MONGODB_OPLOG_URL', value : `mongodb://${data.username}:${data.password}@mongodb:27017/local?authSource=${data.database}` }); env.push({ name: 'CLOUDRON_MONGODB_OPLOG_URL', value : `mongodb://${data.username}:${data.password}@mongodb:27017/local?authSource=${data.database}` });
} }
debug('Setting mongodb addon config to %j', env); log('Setting mongodb addon config to %j', env);
await addonConfigs.set(app.id, 'mongodb', env); await addonConfigs.set(app.id, 'mongodb', env);
} }
@@ -1141,7 +1141,7 @@ async function backupMongoDb(app, options) {
assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof app, 'object');
assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof options, 'object');
debug('Backing up mongodb'); log('Backing up mongodb');
if (!await hasAVX()) throw new BoxError(BoxError.ADDONS_ERROR, 'Error backing up MongoDB. CPU has no AVX support'); if (!await hasAVX()) throw new BoxError(BoxError.ADDONS_ERROR, 'Error backing up MongoDB. CPU has no AVX support');
@@ -1159,7 +1159,7 @@ async function restoreMongoDb(app, options) {
assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof app, 'object');
assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof options, 'object');
debug('restoreMongoDb'); log('restoreMongoDb');
if (!await hasAVX()) throw new BoxError(BoxError.ADDONS_ERROR, 'Error restoring mongodb. CPU has no AVX support'); if (!await hasAVX()) throw new BoxError(BoxError.ADDONS_ERROR, 'Error restoring mongodb. CPU has no AVX support');
@@ -1193,7 +1193,7 @@ async function startGraphite(existingInfra) {
const upgrading = existingInfra.version !== 'none' && requiresUpgrade(existingInfra.images.graphite, image); const upgrading = existingInfra.version !== 'none' && requiresUpgrade(existingInfra.images.graphite, image);
if (upgrading) debug('startGraphite: graphite will be upgraded'); if (upgrading) log('startGraphite: graphite will be upgraded');
const readOnly = !serviceConfig.recoveryMode ? '--read-only' : ''; const readOnly = !serviceConfig.recoveryMode ? '--read-only' : '';
const cmd = serviceConfig.recoveryMode ? '/bin/bash -c \'echo "Debug mode. Sleeping" && sleep infinity\'' : ''; const cmd = serviceConfig.recoveryMode ? '/bin/bash -c \'echo "Debug mode. Sleeping" && sleep infinity\'' : '';
@@ -1213,13 +1213,13 @@ async function startGraphite(existingInfra) {
--label isCloudronManaged=true \ --label isCloudronManaged=true \
${readOnly} -v /tmp -v /run ${image} ${cmd}`; ${readOnly} -v /tmp -v /run ${image} ${cmd}`;
debug('startGraphite: stopping and deleting previous graphite container'); log('startGraphite: stopping and deleting previous graphite container');
await docker.stopContainer('graphite'); await docker.stopContainer('graphite');
await docker.deleteContainer('graphite'); await docker.deleteContainer('graphite');
if (upgrading) await shell.sudo([ RMADDONDIR_CMD, 'graphite' ], {}); if (upgrading) await shell.sudo([ RMADDONDIR_CMD, 'graphite' ], {});
debug('startGraphite: starting graphite container'); log('startGraphite: starting graphite container');
await shell.bash(runCmd, { encoding: 'utf8' }); await shell.bash(runCmd, { encoding: 'utf8' });
if (existingInfra.version !== 'none' && existingInfra.images.graphite !== image) await docker.deleteImage(existingInfra.images.graphite); if (existingInfra.version !== 'none' && existingInfra.images.graphite !== image) await docker.deleteImage(existingInfra.images.graphite);
@@ -1229,7 +1229,7 @@ async function setupProxyAuth(app, options) {
assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof app, 'object');
assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof options, 'object');
debug('Setting up proxyAuth'); log('Setting up proxyAuth');
const enabled = app.sso && app.manifest.addons && app.manifest.addons.proxyAuth; const enabled = app.sso && app.manifest.addons && app.manifest.addons.proxyAuth;
@@ -1238,7 +1238,7 @@ async function setupProxyAuth(app, options) {
const env = [ { name: 'CLOUDRON_PROXY_AUTH', value: '1' } ]; const env = [ { name: 'CLOUDRON_PROXY_AUTH', value: '1' } ];
await addonConfigs.set(app.id, 'proxyauth', env); await addonConfigs.set(app.id, 'proxyauth', env);
debug('Creating OpenID client for proxyAuth'); log('Creating OpenID client for proxyAuth');
const proxyAuthClientId = `${app.id}-proxyauth`; const proxyAuthClientId = `${app.id}-proxyauth`;
const result = await oidcClients.get(proxyAuthClientId); const result = await oidcClients.get(proxyAuthClientId);
@@ -1263,7 +1263,7 @@ async function teardownProxyAuth(app, options) {
await addonConfigs.unset(app.id, 'proxyauth'); await addonConfigs.unset(app.id, 'proxyauth');
debug('Deleting OpenID client for proxyAuth'); log('Deleting OpenID client for proxyAuth');
const proxyAuthClientId = `${app.id}-proxyauth`; const proxyAuthClientId = `${app.id}-proxyauth`;
@@ -1275,7 +1275,7 @@ async function setupDocker(app, options) {
assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof app, 'object');
assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof options, 'object');
debug('Setting up docker'); log('Setting up docker');
const env = [ { name: 'CLOUDRON_DOCKER_HOST', value: `tcp://172.18.0.1:${constants.DOCKER_PROXY_PORT}` } ]; const env = [ { name: 'CLOUDRON_DOCKER_HOST', value: `tcp://172.18.0.1:${constants.DOCKER_PROXY_PORT}` } ];
await addonConfigs.set(app.id, 'docker', env); await addonConfigs.set(app.id, 'docker', env);
@@ -1343,7 +1343,7 @@ async function setupRedis(app, options) {
if (inspectError) { if (inspectError) {
await shell.bash(runCmd, { encoding: 'utf8' }); await shell.bash(runCmd, { encoding: 'utf8' });
} else { // fast path } else { // fast path
debug(`Re-using existing redis container with state: ${JSON.stringify(result.State)}`); log(`Re-using existing redis container with state: ${JSON.stringify(result.State)}`);
} }
if (isAppRunning && !recoveryMode) await waitForContainer(`redis-${app.id}`, 'CLOUDRON_REDIS_TOKEN'); if (isAppRunning && !recoveryMode) await waitForContainer(`redis-${app.id}`, 'CLOUDRON_REDIS_TOKEN');
@@ -1353,7 +1353,7 @@ async function clearRedis(app, options) {
assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof app, 'object');
assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof options, 'object');
debug('Clearing redis'); log('Clearing redis');
const disabled = app.manifest.addons.redis.optional && !app.enableRedis; const disabled = app.manifest.addons.redis.optional && !app.enableRedis;
if (disabled) return; if (disabled) return;
@@ -1377,7 +1377,7 @@ async function teardownRedis(app, options) {
if (error) throw new BoxError(BoxError.FS_ERROR, `Error removing redis data: ${error.message}`); if (error) throw new BoxError(BoxError.FS_ERROR, `Error removing redis data: ${error.message}`);
safe.fs.rmSync(path.join(paths.LOG_DIR, `redis-${app.id}`), { recursive: true, force: true }); safe.fs.rmSync(path.join(paths.LOG_DIR, `redis-${app.id}`), { recursive: true, force: true });
if (safe.error) debug('teardownRedis: cannot cleanup logs: %o', safe.error); if (safe.error) log('teardownRedis: cannot cleanup logs: %o', safe.error);
await addonConfigs.unset(app.id, 'redis'); await addonConfigs.unset(app.id, 'redis');
} }
@@ -1389,7 +1389,7 @@ async function backupRedis(app, options) {
const disabled = app.manifest.addons.redis.optional && !app.enableRedis; const disabled = app.manifest.addons.redis.optional && !app.enableRedis;
if (disabled) return; if (disabled) return;
debug('Backing up redis'); log('Backing up redis');
const result = await getContainerDetails('redis-' + app.id, 'CLOUDRON_REDIS_TOKEN'); const result = await getContainerDetails('redis-' + app.id, 'CLOUDRON_REDIS_TOKEN');
const request = http.request(`http://${result.ip}:3000/backup?access_token=${result.token}`, { method: 'POST' }); const request = http.request(`http://${result.ip}:3000/backup?access_token=${result.token}`, { method: 'POST' });
@@ -1412,11 +1412,11 @@ async function startRedis(existingInfra) {
if (upgrading) await backupRedis(app, {}); if (upgrading) await backupRedis(app, {});
debug(`startRedis: stopping and deleting previous redis container ${redisName}`); log(`startRedis: stopping and deleting previous redis container ${redisName}`);
await docker.stopContainer(redisName); await docker.stopContainer(redisName);
await docker.deleteContainer(redisName); await docker.deleteContainer(redisName);
debug(`startRedis: starting redis container ${redisName}`); log(`startRedis: starting redis container ${redisName}`);
await setupRedis(app, app.manifest.addons.redis); // starts the container await setupRedis(app, app.manifest.addons.redis); // starts the container
} }
@@ -1432,7 +1432,7 @@ async function restoreRedis(app, options) {
const disabled = app.manifest.addons.redis.optional && !app.enableRedis; const disabled = app.manifest.addons.redis.optional && !app.enableRedis;
if (disabled) return; if (disabled) return;
debug('Restoring redis'); log('Restoring redis');
const result = await getContainerDetails('redis-' + app.id, 'CLOUDRON_REDIS_TOKEN'); const result = await getContainerDetails('redis-' + app.id, 'CLOUDRON_REDIS_TOKEN');
const request = http.request(`http://${result.ip}:3000/restore?access_token=${result.token}`, { method: 'POST' }); const request = http.request(`http://${result.ip}:3000/restore?access_token=${result.token}`, { method: 'POST' });
@@ -1445,7 +1445,7 @@ async function setupTls(app, options) {
assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof options, 'object');
if (!safe.fs.mkdirSync(`${paths.PLATFORM_DATA_DIR}/tls/${app.id}`, { recursive: true })) { if (!safe.fs.mkdirSync(`${paths.PLATFORM_DATA_DIR}/tls/${app.id}`, { recursive: true })) {
debug('Error creating tls directory'); log('Error creating tls directory');
throw new BoxError(BoxError.FS_ERROR, safe.error.message); throw new BoxError(BoxError.FS_ERROR, safe.error.message);
} }
} }
@@ -1483,7 +1483,7 @@ async function statusDocker() {
async function restartDocker() { async function restartDocker() {
const [error] = await safe(shell.sudo([ RESTART_SERVICE_CMD, 'docker' ], {})); const [error] = await safe(shell.sudo([ RESTART_SERVICE_CMD, 'docker' ], {}));
if (error) debug(`restartDocker: error restarting docker. ${error.message}`); if (error) log(`restartDocker: error restarting docker. ${error.message}`);
} }
async function statusUnbound() { async function statusUnbound() {
@@ -1493,13 +1493,13 @@ async function statusUnbound() {
const [digError, digResult] = await safe(dig.resolve('ipv4.api.cloudron.io', 'A', { timeout: 10000 })); const [digError, digResult] = await safe(dig.resolve('ipv4.api.cloudron.io', 'A', { timeout: 10000 }));
if (!digError && Array.isArray(digResult) && digResult.length !== 0) return { status: SERVICE_STATUS_ACTIVE }; if (!digError && Array.isArray(digResult) && digResult.length !== 0) return { status: SERVICE_STATUS_ACTIVE };
debug('statusUnbound: unbound is up, but failed to resolve ipv4.api.cloudron.io . %o %j', digError, digResult); log('statusUnbound: unbound is up, but failed to resolve ipv4.api.cloudron.io . %o %j', digError, digResult);
return { status: SERVICE_STATUS_STARTING }; return { status: SERVICE_STATUS_STARTING };
} }
async function restartUnbound() { async function restartUnbound() {
const [error] = await safe(shell.sudo([ RESTART_SERVICE_CMD, 'unbound' ], {})); const [error] = await safe(shell.sudo([ RESTART_SERVICE_CMD, 'unbound' ], {}));
if (error) debug(`restartDocker: error restarting unbound. ${error.message}`); if (error) log(`restartDocker: error restarting unbound. ${error.message}`);
} }
async function statusNginx() { async function statusNginx() {
@@ -1509,7 +1509,7 @@ async function statusNginx() {
async function restartNginx() { async function restartNginx() {
const [error] = await safe(shell.sudo([ RESTART_SERVICE_CMD, 'nginx' ], {})); const [error] = await safe(shell.sudo([ RESTART_SERVICE_CMD, 'nginx' ], {}));
if (error) debug(`restartNginx: error restarting unbound. ${error.message}`); if (error) log(`restartNginx: error restarting unbound. ${error.message}`);
} }
async function statusGraphite() { async function statusGraphite() {
@@ -1676,7 +1676,7 @@ async function getServiceLogs(id, options) {
throw new BoxError(BoxError.NOT_FOUND, 'Service not found'); throw new BoxError(BoxError.NOT_FOUND, 'Service not found');
} }
debug(`getServiceLogs: getting logs for ${name}`); log(`getServiceLogs: getting logs for ${name}`);
let cp; let cp;
@@ -1730,7 +1730,7 @@ async function applyMemoryLimit(id) {
memoryLimit = serviceConfig && serviceConfig.memoryLimit ? serviceConfig.memoryLimit : APP_SERVICES[name].defaultMemoryLimit; memoryLimit = serviceConfig && serviceConfig.memoryLimit ? serviceConfig.memoryLimit : APP_SERVICES[name].defaultMemoryLimit;
} else if (SERVICES[name]) { } else if (SERVICES[name]) {
if (name === 'mongodb' && !await hasAVX()) { if (name === 'mongodb' && !await hasAVX()) {
debug('applyMemoryLimit: skipping mongodb because CPU does not have AVX'); log('applyMemoryLimit: skipping mongodb because CPU does not have AVX');
return; return;
} }
@@ -1743,12 +1743,12 @@ async function applyMemoryLimit(id) {
if (LAZY_SERVICES.includes(name)) { if (LAZY_SERVICES.includes(name)) {
const [error, container] = await safe(docker.inspect(containerName)); const [error, container] = await safe(docker.inspect(containerName));
if (error || !container.State?.Running) { if (error || !container.State?.Running) {
debug(`applyMemoryLimit: skipping ${containerName} (not running)`); log(`applyMemoryLimit: skipping ${containerName} (not running)`);
return; return;
} }
} }
debug(`applyMemoryLimit: ${containerName} ${JSON.stringify(serviceConfig)}`); log(`applyMemoryLimit: ${containerName} ${JSON.stringify(serviceConfig)}`);
await docker.update(containerName, memoryLimit); await docker.update(containerName, memoryLimit);
} }
@@ -1766,7 +1766,7 @@ async function applyServiceLimits() {
changed = true; changed = true;
} }
safe(applyMemoryLimit(id), { debug }); safe(applyMemoryLimit(id), { debug: log });
} }
if (changed) await settings.setJson(settings.SERVICES_CONFIG_KEY, servicesConfig); if (changed) await settings.setJson(settings.SERVICES_CONFIG_KEY, servicesConfig);
@@ -1783,7 +1783,7 @@ async function applyServiceLimits() {
await apps.update(app.id, { servicesConfig: app.servicesConfig }); await apps.update(app.id, { servicesConfig: app.servicesConfig });
} }
safe(applyMemoryLimit(`redis:${app.id}`), { debug }); safe(applyMemoryLimit(`redis:${app.id}`), { debug: log });
} }
} }
@@ -1797,7 +1797,7 @@ async function startTurn(existingInfra) {
let turnSecret = await blobs.getString(blobs.ADDON_TURN_SECRET); let turnSecret = await blobs.getString(blobs.ADDON_TURN_SECRET);
if (!turnSecret) { if (!turnSecret) {
debug('startTurn: generating turn secret'); log('startTurn: generating turn secret');
turnSecret = 'a' + crypto.randomBytes(15).toString('hex'); // prefix with a to ensure string starts with a letter turnSecret = 'a' + crypto.randomBytes(15).toString('hex'); // prefix with a to ensure string starts with a letter
await blobs.setString(blobs.ADDON_TURN_SECRET, turnSecret); await blobs.setString(blobs.ADDON_TURN_SECRET, turnSecret);
} }
@@ -1825,11 +1825,11 @@ async function startTurn(existingInfra) {
--label isCloudronManaged=true \ --label isCloudronManaged=true \
${readOnly} -v /tmp -v /run ${image} ${cmd}`; ${readOnly} -v /tmp -v /run ${image} ${cmd}`;
debug('startTurn: stopping and deleting previous turn container'); log('startTurn: stopping and deleting previous turn container');
await docker.stopContainer('turn'); await docker.stopContainer('turn');
await docker.deleteContainer('turn'); await docker.deleteContainer('turn');
debug('startTurn: starting turn container'); log('startTurn: starting turn container');
await shell.bash(runCmd, { encoding: 'utf8' }); await shell.bash(runCmd, { encoding: 'utf8' });
if (existingInfra.version !== 'none' && existingInfra.images.turn !== image) await docker.deleteImage(existingInfra.images.turn); if (existingInfra.version !== 'none' && existingInfra.images.turn !== image) await docker.deleteImage(existingInfra.images.turn);
@@ -1877,7 +1877,7 @@ async function rebuildService(id, auditSource) {
// nothing to rebuild for now. // nothing to rebuild for now.
} }
safe(applyMemoryLimit(id), { debug }); // do this in background. ok to fail safe(applyMemoryLimit(id), { debug: log }); // do this in background. ok to fail
await eventlog.add(eventlog.ACTION_SERVICE_REBUILD, auditSource, { id }); await eventlog.add(eventlog.ACTION_SERVICE_REBUILD, auditSource, { id });
} }
@@ -1912,13 +1912,13 @@ async function configureService(id, data, auditSource) {
throw new BoxError(BoxError.NOT_FOUND, 'No such service'); throw new BoxError(BoxError.NOT_FOUND, 'No such service');
} }
debug(`configureService: ${id} rebuild=${needsRebuild}`); log(`configureService: ${id} rebuild=${needsRebuild}`);
// do this in background // do this in background
if (needsRebuild) { if (needsRebuild) {
safe(rebuildService(id, auditSource), { debug }); safe(rebuildService(id, auditSource), { debug: log });
} else { } else {
safe(applyMemoryLimit(id), { debug }); safe(applyMemoryLimit(id), { debug: log });
} }
await eventlog.add(eventlog.ACTION_SERVICE_CONFIGURE, auditSource, { id, data }); await eventlog.add(eventlog.ACTION_SERVICE_CONFIGURE, auditSource, { id, data });
@@ -1950,14 +1950,14 @@ async function startServices(existingInfra, progressCallback) {
await fn(existingInfra); await fn(existingInfra);
} }
safe(applyServiceLimits(), { debug }); safe(applyServiceLimits(), { debug: log });
} }
async function teardownOauth(app, options) { async function teardownOauth(app, options) {
assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof app, 'object');
assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof options, 'object');
debug('teardownOauth'); log('teardownOauth');
await addonConfigs.unset(app.id, 'oauth'); await addonConfigs.unset(app.id, 'oauth');
} }
@@ -1968,7 +1968,7 @@ async function setupOidc(app, options) {
if (!app.sso) return; if (!app.sso) return;
debug('Setting up OIDC'); log('Setting up OIDC');
const oidcAddonClientId = `${app.id}-oidc`; const oidcAddonClientId = `${app.id}-oidc`;
const [error, result] = await safe(oidcClients.get(oidcAddonClientId)); const [error, result] = await safe(oidcClients.get(oidcAddonClientId));
@@ -1993,7 +1993,7 @@ async function teardownOidc(app, options) {
assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof app, 'object');
assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof options, 'object');
debug('Tearing down OIDC'); log('Tearing down OIDC');
const oidcAddonClientId = `${app.id}-oidc`; const oidcAddonClientId = `${app.id}-oidc`;
@@ -2047,7 +2047,7 @@ async function moveDataDir(app, targetVolumeId, targetVolumePrefix) {
const resolvedSourceDir = await apps.getStorageDir(app); const resolvedSourceDir = await apps.getStorageDir(app);
const resolvedTargetDir = await apps.getStorageDir(Object.assign({}, app, { storageVolumeId: targetVolumeId, storageVolumePrefix: targetVolumePrefix })); const resolvedTargetDir = await apps.getStorageDir(Object.assign({}, app, { storageVolumeId: targetVolumeId, storageVolumePrefix: targetVolumePrefix }));
debug(`moveDataDir: migrating data from ${resolvedSourceDir} to ${resolvedTargetDir}`); log(`moveDataDir: migrating data from ${resolvedSourceDir} to ${resolvedTargetDir}`);
if (resolvedSourceDir !== resolvedTargetDir) { if (resolvedSourceDir !== resolvedTargetDir) {
const [error] = await safe(shell.sudo([ MV_VOLUME_CMD, resolvedSourceDir, resolvedTargetDir ], {})); const [error] = await safe(shell.sudo([ MV_VOLUME_CMD, resolvedSourceDir, resolvedTargetDir ], {}));
@@ -2203,12 +2203,12 @@ async function setupAddons(app, addons) {
if (!addons) return; if (!addons) return;
debug('setupAddons: Setting up %j', Object.keys(addons)); log('setupAddons: Setting up %j', Object.keys(addons));
for (const addon of Object.keys(addons)) { for (const addon of Object.keys(addons)) {
if (!(addon in ADDONS)) throw new BoxError(BoxError.NOT_FOUND, `No such addon: ${addon}`); if (!(addon in ADDONS)) throw new BoxError(BoxError.NOT_FOUND, `No such addon: ${addon}`);
debug(`setupAddons: setting up addon ${addon} with options ${JSON.stringify(addons[addon])}`); log(`setupAddons: setting up addon ${addon} with options ${JSON.stringify(addons[addon])}`);
await ADDONS[addon].setup(app, addons[addon]); await ADDONS[addon].setup(app, addons[addon]);
} }
@@ -2220,12 +2220,12 @@ async function teardownAddons(app, addons) {
if (!addons) return; if (!addons) return;
debug('teardownAddons: Tearing down %j', Object.keys(addons)); log('teardownAddons: Tearing down %j', Object.keys(addons));
for (const addon of Object.keys(addons)) { for (const addon of Object.keys(addons)) {
if (!(addon in ADDONS)) throw new BoxError(BoxError.NOT_FOUND, `No such addon: ${addon}`); if (!(addon in ADDONS)) throw new BoxError(BoxError.NOT_FOUND, `No such addon: ${addon}`);
debug(`teardownAddons: Tearing down addon ${addon} with options ${JSON.stringify(addons[addon])}`); log(`teardownAddons: Tearing down addon ${addon} with options ${JSON.stringify(addons[addon])}`);
await ADDONS[addon].teardown(app, addons[addon]); await ADDONS[addon].teardown(app, addons[addon]);
} }
@@ -2237,7 +2237,7 @@ async function backupAddons(app, addons) {
if (!addons) return; if (!addons) return;
debug('backupAddons: backing up %j', Object.keys(addons)); log('backupAddons: backing up %j', Object.keys(addons));
for (const addon of Object.keys(addons)) { for (const addon of Object.keys(addons)) {
if (!(addon in ADDONS)) throw new BoxError(BoxError.NOT_FOUND, `No such addon: ${addon}`); if (!(addon in ADDONS)) throw new BoxError(BoxError.NOT_FOUND, `No such addon: ${addon}`);
@@ -2252,7 +2252,7 @@ async function clearAddons(app, addons) {
if (!addons) return; if (!addons) return;
debug('clearAddons: clearing %j', Object.keys(addons)); log('clearAddons: clearing %j', Object.keys(addons));
for (const addon of Object.keys(addons)) { for (const addon of Object.keys(addons)) {
if (!(addon in ADDONS)) throw new BoxError(BoxError.NOT_FOUND, `No such addon: ${addon}`); if (!(addon in ADDONS)) throw new BoxError(BoxError.NOT_FOUND, `No such addon: ${addon}`);
@@ -2268,7 +2268,7 @@ async function restoreAddons(app, addons) {
if (!addons) return; if (!addons) return;
debug('restoreAddons: restoring %j', Object.keys(addons)); log('restoreAddons: restoring %j', Object.keys(addons));
for (const addon of Object.keys(addons)) { for (const addon of Object.keys(addons)) {
if (!(addon in ADDONS)) throw new BoxError(BoxError.NOT_FOUND, `No such addon: ${addon}`); if (!(addon in ADDONS)) throw new BoxError(BoxError.NOT_FOUND, `No such addon: ${addon}`);

16
src/sftp.js
View File

@@ -3,7 +3,7 @@ import assert from 'node:assert';
import blobs from './blobs.js'; import blobs from './blobs.js';
import BoxError from './boxerror.js'; import BoxError from './boxerror.js';
import constants from './constants.js'; import constants from './constants.js';
import debugModule from 'debug'; import logger from './logger.js';
import docker from './docker.js'; import docker from './docker.js';
import hat from './hat.js'; import hat from './hat.js';
import infra from './infra_version.js'; import infra from './infra_version.js';
@@ -15,7 +15,7 @@ import services from './services.js';
import shellModule from './shell.js'; import shellModule from './shell.js';
import volumes from './volumes.js'; import volumes from './volumes.js';
const debug = debugModule('box:sftp'); const { log, trace } = logger('sftp');
const shell = shellModule('sftp'); const shell = shellModule('sftp');
const DEFAULT_MEMORY_LIMIT = 256 * 1024 * 1024; const DEFAULT_MEMORY_LIMIT = 256 * 1024 * 1024;
@@ -29,7 +29,7 @@ async function ensureKeys() {
const privateKeyFile = path.join(paths.SFTP_KEYS_DIR, `ssh_host_${keyType}_key`); const privateKeyFile = path.join(paths.SFTP_KEYS_DIR, `ssh_host_${keyType}_key`);
if (!privateKey || !publicKey) { if (!privateKey || !publicKey) {
debug(`ensureSecrets: generating new sftp keys of type ${keyType}`); log(`ensureSecrets: generating new sftp keys of type ${keyType}`);
safe.fs.unlinkSync(publicKeyFile); safe.fs.unlinkSync(publicKeyFile);
safe.fs.unlinkSync(privateKeyFile); safe.fs.unlinkSync(privateKeyFile);
const [error] = await safe(shell.spawn('ssh-keygen', ['-m', 'PEM', '-t', keyType, '-f', `${paths.SFTP_KEYS_DIR}/ssh_host_${keyType}_key`, '-q', '-N', ''], {})); const [error] = await safe(shell.spawn('ssh-keygen', ['-m', 'PEM', '-t', keyType, '-f', `${paths.SFTP_KEYS_DIR}/ssh_host_${keyType}_key`, '-q', '-N', ''], {}));
@@ -48,7 +48,7 @@ async function ensureKeys() {
async function start(existingInfra) { async function start(existingInfra) {
assert.strictEqual(typeof existingInfra, 'object'); assert.strictEqual(typeof existingInfra, 'object');
debug('start: re-creating container'); log('start: re-creating container');
const serviceConfig = await services.getServiceConfig('sftp'); const serviceConfig = await services.getServiceConfig('sftp');
const image = infra.images.sftp; const image = infra.images.sftp;
@@ -70,7 +70,7 @@ async function start(existingInfra) {
const hostDir = await apps.getStorageDir(app), mountDir = `/mnt/app-${app.id}`; // see also sftp:userSearchSftp const hostDir = await apps.getStorageDir(app), mountDir = `/mnt/app-${app.id}`; // see also sftp:userSearchSftp
if (hostDir === null || !safe.fs.existsSync(hostDir)) { // this can fail if external mount does not have permissions for yellowtent user if (hostDir === null || !safe.fs.existsSync(hostDir)) { // this can fail if external mount does not have permissions for yellowtent user
// do not create host path when cloudron is restoring. this will then create dir with root perms making restore logic fail // do not create host path when cloudron is restoring. this will then create dir with root perms making restore logic fail
debug(`Ignoring app data dir ${hostDir} for ${app.id} since it does not exist`); log(`Ignoring app data dir ${hostDir} for ${app.id} since it does not exist`);
continue; continue;
} }
@@ -84,7 +84,7 @@ async function start(existingInfra) {
if (mounts.isManagedProvider(volume.mountType)) continue; // skip managed volume. these are acessed via /mnt/volumes mount above if (mounts.isManagedProvider(volume.mountType)) continue; // skip managed volume. these are acessed via /mnt/volumes mount above
if (!safe.fs.existsSync(volume.hostPath)) { if (!safe.fs.existsSync(volume.hostPath)) {
debug(`Ignoring volume host path ${volume.hostPath} since it does not exist`); log(`Ignoring volume host path ${volume.hostPath} since it does not exist`);
continue; continue;
} }
@@ -118,11 +118,11 @@ async function start(existingInfra) {
--label isCloudronManaged=true \ --label isCloudronManaged=true \
${readOnly} -v /tmp -v /run ${image} ${cmd}`; ${readOnly} -v /tmp -v /run ${image} ${cmd}`;
debug('startSftp: stopping and deleting previous sftp container'); log('startSftp: stopping and deleting previous sftp container');
await docker.stopContainer('sftp'); await docker.stopContainer('sftp');
await docker.deleteContainer('sftp'); await docker.deleteContainer('sftp');
debug('startSftp: starting sftp container'); log('startSftp: starting sftp container');
await shell.bash(runCmd, { encoding: 'utf8' }); await shell.bash(runCmd, { encoding: 'utf8' });
if (existingInfra.version !== 'none' && existingInfra.images.sftp !== image) await docker.deleteImage(existingInfra.images.sftp); if (existingInfra.version !== 'none' && existingInfra.images.sftp !== image) await docker.deleteImage(existingInfra.images.sftp);

20
src/shell.js
View File

@@ -1,12 +1,12 @@
import assert from 'node:assert'; import assert from 'node:assert';
import BoxError from './boxerror.js'; import BoxError from './boxerror.js';
import child_process from 'node:child_process'; import child_process from 'node:child_process';
import debugModule from 'debug'; import logger from './logger.js';
import path from 'node:path'; import path from 'node:path';
import safe from 'safetydance'; import safe from 'safetydance';
import _ from './underscore.js'; import _ from './underscore.js';
const debug = debugModule('box:shell'); const { log, trace } = logger('shell');
function lineCount(buffer) { function lineCount(buffer) {
assert(Buffer.isBuffer(buffer)); assert(Buffer.isBuffer(buffer));
@@ -30,7 +30,7 @@ function spawn(tag, file, args, options) {
assert(Array.isArray(args)); assert(Array.isArray(args));
assert.strictEqual(typeof options, 'object'); // note: spawn() has no encoding option of it's own assert.strictEqual(typeof options, 'object'); // note: spawn() has no encoding option of it's own
debug(`${tag}: ${file} ${args.join(' ').replace(/\n/g, '\\n')}`); log(`${tag}: ${file} ${args.join(' ').replace(/\n/g, '\\n')}`);
const maxLines = options.maxLines || Number.MAX_SAFE_INTEGER; const maxLines = options.maxLines || Number.MAX_SAFE_INTEGER;
const logger = options.logger || null; const logger = options.logger || null;
@@ -79,26 +79,26 @@ function spawn(tag, file, args, options) {
e.timedOut = timedOut; e.timedOut = timedOut;
e.terminated = terminated; e.terminated = terminated;
debug(`${tag}: ${file} ${args.join(' ').replace(/\n/g, '\\n')} errored`, e); log(`${tag}: ${file} ${args.join(' ').replace(/\n/g, '\\n')} errored`, e);
reject(e); reject(e);
}); });
cp.on('error', function (error) { // when the command itself could not be started cp.on('error', function (error) { // when the command itself could not be started
debug(`${tag}: ${file} ${args.join(' ').replace(/\n/g, '\\n')} errored`, error); log(`${tag}: ${file} ${args.join(' ').replace(/\n/g, '\\n')} errored`, error);
}); });
cp.terminate = function () { cp.terminate = function () {
terminated = true; terminated = true;
// many approaches to kill sudo launched process failed. we now have a sudo wrapper to kill the full tree // many approaches to kill sudo launched process failed. we now have a sudo wrapper to kill the full tree
child_process.execFile('/usr/bin/sudo', [ KILL_CHILD_CMD, cp.pid, process.pid ], { encoding: 'utf8' }, (error, stdout, stderr) => { child_process.execFile('/usr/bin/sudo', [ KILL_CHILD_CMD, cp.pid, process.pid ], { encoding: 'utf8' }, (error, stdout, stderr) => {
if (error) debug(`${tag}: failed to kill children`, stdout, stderr); if (error) log(`${tag}: failed to kill children`, stdout, stderr);
else debug(`${tag}: terminated ${cp.pid}`, stdout, stderr); else log(`${tag}: terminated ${cp.pid}`, stdout, stderr);
}); });
}; };
abortSignal?.addEventListener('abort', () => { abortSignal?.addEventListener('abort', () => {
debug(`${tag}: aborting ${cp.pid}`); log(`${tag}: aborting ${cp.pid}`);
cp.terminate(); cp.terminate();
}, { once: true }); }, { once: true });
@@ -106,10 +106,10 @@ function spawn(tag, file, args, options) {
if (options.timeout) { if (options.timeout) {
killTimerId = setTimeout(async () => { killTimerId = setTimeout(async () => {
debug(`${tag}: timedout`); log(`${tag}: timedout`);
timedOut = true; timedOut = true;
if (typeof options.onTimeout !== 'function') return cp.terminate(); if (typeof options.onTimeout !== 'function') return cp.terminate();
await safe(options.onTimeout(), { debug }); await safe(options.onTimeout(), { debug: log });
}, options.timeout); }, options.timeout);
} }

18
src/storage/filesystem.js
View File

@@ -1,7 +1,7 @@
import assert from 'node:assert'; import assert from 'node:assert';
import BoxError from '../boxerror.js'; import BoxError from '../boxerror.js';
import crypto from 'crypto'; import crypto from 'crypto';
import debugModule from 'debug'; import logger from '../logger.js';
import df from '../df.js'; import df from '../df.js';
import fs from 'node:fs'; import fs from 'node:fs';
import mounts from '../mounts.js'; import mounts from '../mounts.js';
@@ -11,7 +11,7 @@ import safe from 'safetydance';
import shellModule from '../shell.js'; import shellModule from '../shell.js';
import _ from '../underscore.js'; import _ from '../underscore.js';
const debug = debugModule('box:storage/filesystem'); const { log, trace } = logger('storage/filesystem');
const shell = shellModule('filesystem'); const shell = shellModule('filesystem');
@@ -111,7 +111,7 @@ async function download(config, remotePath) {
assert.strictEqual(typeof remotePath, 'string'); assert.strictEqual(typeof remotePath, 'string');
const fullRemotePath = path.join(getRootPath(config), remotePath); const fullRemotePath = path.join(getRootPath(config), remotePath);
debug(`download: ${fullRemotePath}`); log(`download: ${fullRemotePath}`);
if (!safe.fs.existsSync(fullRemotePath)) throw new BoxError(BoxError.NOT_FOUND, `File not found: ${fullRemotePath}`); if (!safe.fs.existsSync(fullRemotePath)) throw new BoxError(BoxError.NOT_FOUND, `File not found: ${fullRemotePath}`);
@@ -208,7 +208,7 @@ async function copyInternal(config, fromPath, toPath, options, progressCallback)
safe.fs.unlinkSync(identityFilePath); safe.fs.unlinkSync(identityFilePath);
if (!remoteCopyError) return; if (!remoteCopyError) return;
if (remoteCopyError.code === 255) throw new BoxError(BoxError.EXTERNAL_ERROR, `SSH connection error: ${remoteCopyError.message}`); // do not attempt fallback copy for ssh errors if (remoteCopyError.code === 255) throw new BoxError(BoxError.EXTERNAL_ERROR, `SSH connection error: ${remoteCopyError.message}`); // do not attempt fallback copy for ssh errors
debug('SSH remote copy failed, trying sshfs copy'); // this can happen for sshfs mounted windows server log('SSH remote copy failed, trying sshfs copy'); // this can happen for sshfs mounted windows server
} }
const [copyError] = await safe(shell.spawn('cp', [ cpOptions, fullFromPath, fullToPath ], {})); const [copyError] = await safe(shell.spawn('cp', [ cpOptions, fullFromPath, fullToPath ], {}));
@@ -271,7 +271,7 @@ async function removeDir(config, limits, remotePathPrefix, progressCallback) {
safe.fs.unlinkSync(identityFilePath); safe.fs.unlinkSync(identityFilePath);
if (!remoteRmError) return; if (!remoteRmError) return;
if (remoteRmError.code === 255) throw new BoxError(BoxError.EXTERNAL_ERROR, `SSH connection error: ${remoteRmError.message}`); // do not attempt fallback copy for ssh errors if (remoteRmError.code === 255) throw new BoxError(BoxError.EXTERNAL_ERROR, `SSH connection error: ${remoteRmError.message}`); // do not attempt fallback copy for ssh errors
debug('SSH remote rm failed, trying sshfs rm'); // this can happen for sshfs mounted windows server log('SSH remote rm failed, trying sshfs rm'); // this can happen for sshfs mounted windows server
} }
const [error] = await safe(shell.spawn('rm', [ '-rf', fullPathPrefix ], {})); const [error] = await safe(shell.spawn('rm', [ '-rf', fullPathPrefix ], {}));
@@ -311,13 +311,13 @@ function mountObjectFromConfig(config) {
async function setup(config) { async function setup(config) {
assert.strictEqual(typeof config, 'object'); assert.strictEqual(typeof config, 'object');
debug('setup: removing old storage configuration'); log('setup: removing old storage configuration');
if (!mounts.isManagedProvider(config._provider)) return; if (!mounts.isManagedProvider(config._provider)) return;
const mount = mountObjectFromConfig(config); const mount = mountObjectFromConfig(config);
await safe(mounts.removeMount(mount), { debug }); // ignore error await safe(mounts.removeMount(mount), { debug: log }); // ignore error
debug('setup: setting up new storage configuration'); log('setup: setting up new storage configuration');
await mounts.tryAddMount(mount, { timeout: 10 }); // 10 seconds await mounts.tryAddMount(mount, { timeout: 10 }); // 10 seconds
} }
@@ -326,7 +326,7 @@ async function teardown(config) {
if (!mounts.isManagedProvider(config._provider)) return; if (!mounts.isManagedProvider(config._provider)) return;
await safe(mounts.removeMount(mountObjectFromConfig(config)), { debug }); // ignore error await safe(mounts.removeMount(mountObjectFromConfig(config)), { debug: log }); // ignore error
} }
async function verifyConfig({ id, provider, config }) { async function verifyConfig({ id, provider, config }) {

18
src/storage/gcs.js
View File

@@ -2,13 +2,13 @@ import assert from 'node:assert';
import async from 'async'; import async from 'async';
import BoxError from '../boxerror.js'; import BoxError from '../boxerror.js';
import constants from '../constants.js'; import constants from '../constants.js';
import debugModule from 'debug'; import logger from '../logger.js';
import { Storage as GCS } from '@google-cloud/storage'; import { Storage as GCS } from '@google-cloud/storage';
import path from 'node:path'; import path from 'node:path';
import safe from 'safetydance'; import safe from 'safetydance';
import _ from '../underscore.js'; import _ from '../underscore.js';
const debug = debugModule('box:storage/gcs'); const { log, trace } = logger('storage/gcs');
function getBucket(config) { function getBucket(config) {
@@ -50,7 +50,7 @@ async function upload(config, limits, remotePath) {
const fullRemotePath = path.join(config.prefix, remotePath); const fullRemotePath = path.join(config.prefix, remotePath);
debug(`Uploading to ${fullRemotePath}`); log(`Uploading to ${fullRemotePath}`);
return { return {
createStream() { return getBucket(config).file(fullRemotePath).createWriteStream({ resumable: false }); }, createStream() { return getBucket(config).file(fullRemotePath).createWriteStream({ resumable: false }); },
@@ -91,7 +91,7 @@ async function download(config, remotePath) {
assert.strictEqual(typeof remotePath, 'string'); assert.strictEqual(typeof remotePath, 'string');
const fullRemotePath = path.join(config.prefix, remotePath); const fullRemotePath = path.join(config.prefix, remotePath);
debug(`Download ${fullRemotePath} starting`); log(`Download ${fullRemotePath} starting`);
const file = getBucket(config).file(fullRemotePath); const file = getBucket(config).file(fullRemotePath);
return file.createReadStream(); return file.createReadStream();
@@ -124,7 +124,7 @@ async function copyInternal(config, fullFromPath, fullToPath, progressCallback)
assert.strictEqual(typeof progressCallback, 'function'); assert.strictEqual(typeof progressCallback, 'function');
const [copyError] = await safe(getBucket(config).file(fullFromPath).copy(fullToPath)); const [copyError] = await safe(getBucket(config).file(fullFromPath).copy(fullToPath));
if (copyError) debug('copyBackup: gcs copy error. %o', copyError); if (copyError) log('copyBackup: gcs copy error. %o', copyError);
if (copyError && copyError.code === 404) throw new BoxError(BoxError.NOT_FOUND, 'Old backup not found'); if (copyError && copyError.code === 404) throw new BoxError(BoxError.NOT_FOUND, 'Old backup not found');
if (copyError) throw new BoxError(BoxError.EXTERNAL_ERROR, copyError.message); if (copyError) throw new BoxError(BoxError.EXTERNAL_ERROR, copyError.message);
} }
@@ -175,7 +175,7 @@ async function remove(config, remotePath) {
const fullRemotePath = path.join(config.prefix, remotePath); const fullRemotePath = path.join(config.prefix, remotePath);
const [error] = await safe(getBucket(config).file(fullRemotePath).delete()); const [error] = await safe(getBucket(config).file(fullRemotePath).delete());
if (error) debug('removeBackups: Unable to remove %s (%s). Not fatal.', fullRemotePath, error.message); if (error) log('removeBackups: Unable to remove %s (%s). Not fatal.', fullRemotePath, error.message);
} }
async function removeDir(config, limits, remotePathPrefix, progressCallback) { async function removeDir(config, limits, remotePathPrefix, progressCallback) {
@@ -226,11 +226,11 @@ async function verifyConfig({ id, provider, config }) {
const testFile = bucket.file(path.join(config.prefix, 'snapshot/cloudron-testfile')); const testFile = bucket.file(path.join(config.prefix, 'snapshot/cloudron-testfile'));
const [saveError] = await safe(testFile.save('testfilecontents', { resumable: false })); const [saveError] = await safe(testFile.save('testfilecontents', { resumable: false }));
if (saveError) { // with bad creds, it return status 500 and "Cannot call write after a stream was destroyed" if (saveError) { // with bad creds, it return status 500 and "Cannot call write after a stream was destroyed"
debug('testConfig: failed uploading cloudron-testfile. %o', saveError); log('testConfig: failed uploading cloudron-testfile. %o', saveError);
if (saveError.code == 403 || saveError.code == 404) throw new BoxError(BoxError.BAD_FIELD, saveError.message); if (saveError.code == 403 || saveError.code == 404) throw new BoxError(BoxError.BAD_FIELD, saveError.message);
throw new BoxError(BoxError.EXTERNAL_ERROR, saveError.message); throw new BoxError(BoxError.EXTERNAL_ERROR, saveError.message);
} }
debug('testConfig: uploaded cloudron-testfile'); log('testConfig: uploaded cloudron-testfile');
const query = { prefix: path.join(config.prefix, 'snapshot'), autoPaginate: false, maxResults: 1 }; const query = { prefix: path.join(config.prefix, 'snapshot'), autoPaginate: false, maxResults: 1 };
const [listError] = await safe(bucket.getFiles(query)); const [listError] = await safe(bucket.getFiles(query));
@@ -238,7 +238,7 @@ async function verifyConfig({ id, provider, config }) {
const [delError] = await safe(bucket.file(path.join(config.prefix, 'snapshot/cloudron-testfile')).delete()); const [delError] = await safe(bucket.file(path.join(config.prefix, 'snapshot/cloudron-testfile')).delete());
if (delError) throw new BoxError(BoxError.EXTERNAL_ERROR, delError.message); if (delError) throw new BoxError(BoxError.EXTERNAL_ERROR, delError.message);
debug('testConfig: deleted cloudron-testfile'); log('testConfig: deleted cloudron-testfile');
return _.pick(config, ['projectId', 'credentials', 'bucket', 'prefix']); return _.pick(config, ['projectId', 'credentials', 'bucket', 'prefix']);
} }

20
src/storage/s3.js
View File

@@ -5,7 +5,7 @@ import { ConfiguredRetryStrategy } from '@smithy/util-retry';
import constants from '../constants.js'; import constants from '../constants.js';
import consumers from 'node:stream/consumers'; import consumers from 'node:stream/consumers';
import crypto from 'node:crypto'; import crypto from 'node:crypto';
import debugModule from 'debug'; import logger from '../logger.js';
import http from 'node:http'; import http from 'node:http';
import https from 'node:https'; import https from 'node:https';
import { NodeHttpHandler } from '@smithy/node-http-handler'; import { NodeHttpHandler } from '@smithy/node-http-handler';
@@ -17,7 +17,7 @@ import safe from 'safetydance';
import { Upload } from '@aws-sdk/lib-storage'; import { Upload } from '@aws-sdk/lib-storage';
import _ from '../underscore.js'; import _ from '../underscore.js';
const debug = debugModule('box:storage/s3'); const { log, trace } = logger('storage/s3');
function S3_NOT_FOUND(error) { function S3_NOT_FOUND(error) {
return error instanceof NoSuchKey || error instanceof NoSuchBucket; return error instanceof NoSuchKey || error instanceof NoSuchBucket;
@@ -93,8 +93,8 @@ function createS3Client(config, options) {
const client = constants.TEST ? new globalThis.S3Mock(clientConfig) : new S3(clientConfig); const client = constants.TEST ? new globalThis.S3Mock(clientConfig) : new S3(clientConfig);
// https://github.com/aws/aws-sdk-js-v3/issues/6761#issuecomment-2574480834 // https://github.com/aws/aws-sdk-js-v3/issues/6761#issuecomment-2574480834
// client.middlewareStack.add((next, context) => async (args) => { // client.middlewareStack.add((next, context) => async (args) => {
// debug('AWS SDK context', context.clientName, context.commandName); // log('AWS SDK context', context.clientName, context.commandName);
// debug('AWS SDK request input', JSON.stringify(args.input)); // log('AWS SDK request input', JSON.stringify(args.input));
// const result = await next(args); // const result = await next(args);
// console.log('AWS SDK request output:', result.output); // console.log('AWS SDK request output:', result.output);
// return result; // return result;
@@ -174,7 +174,7 @@ async function upload(config, limits, remotePath) {
}; };
const managedUpload = constants.TEST ? new globalThis.S3MockUpload(options) : new Upload(options); const managedUpload = constants.TEST ? new globalThis.S3MockUpload(options) : new Upload(options);
// managedUpload.on('httpUploadProgress', (progress) => debug(`Upload progress: ${JSON.stringify(progress)}`)); // managedUpload.on('httpUploadProgress', (progress) => log(`Upload progress: ${JSON.stringify(progress)}`));
uploadPromise = managedUpload.done(); uploadPromise = managedUpload.done();
return passThrough; return passThrough;
@@ -183,7 +183,7 @@ async function upload(config, limits, remotePath) {
if (!uploadPromise) return; // stream was never created if (!uploadPromise) return; // stream was never created
const [error/*,data*/] = await safe(uploadPromise); const [error/*,data*/] = await safe(uploadPromise);
if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, `Upload error: code: ${error.Code} message: ${error.message}`); // sometimes message is null if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, `Upload error: code: ${error.Code} message: ${error.message}`); // sometimes message is null
// debug(`Upload finished. ${JSON.stringify(data)}`); // { ETag, $metadata:{httpStatusCode,requestId,attempts,totalRetryDelay},Bucket,Key} // log(`Upload finished. ${JSON.stringify(data)}`); // { ETag, $metadata:{httpStatusCode,requestId,attempts,totalRetryDelay},Bucket,Key}
} }
}; };
} }
@@ -245,7 +245,7 @@ class S3MultipartDownloadStream extends Readable {
if (S3_NOT_FOUND(error)) { if (S3_NOT_FOUND(error)) {
this.destroy(new BoxError(BoxError.NOT_FOUND, `Backup not found: ${this._path}`)); this.destroy(new BoxError(BoxError.NOT_FOUND, `Backup not found: ${this._path}`));
} else { } else {
debug(`download: ${this._path} s3 stream error. %o`, error); log(`download: ${this._path} s3 stream error. %o`, error);
this.destroy(new BoxError(BoxError.EXTERNAL_ERROR, `Error multipartDownload ${this._path}. ${formatError(error)}`)); this.destroy(new BoxError(BoxError.EXTERNAL_ERROR, `Error multipartDownload ${this._path}. ${formatError(error)}`));
} }
} }
@@ -443,7 +443,7 @@ async function cleanup(config, progressCallback) {
for (const multipartUpload of uploads.Uploads) { for (const multipartUpload of uploads.Uploads) {
if (Date.now() - new Date(multipartUpload.Initiated) < 3 * 24 * 60 * 60 * 1000) continue; // 3 days ago if (Date.now() - new Date(multipartUpload.Initiated) < 3 * 24 * 60 * 60 * 1000) continue; // 3 days ago
progressCallback({ message: `Cleaning up multi-part upload uploadId:${multipartUpload.UploadId} key:${multipartUpload.Key}` }); progressCallback({ message: `Cleaning up multi-part upload uploadId:${multipartUpload.UploadId} key:${multipartUpload.Key}` });
await safe(s3.abortMultipartUpload({ Bucket: config.bucket, Key: multipartUpload.Key, UploadId: multipartUpload.UploadId }), { debug }); // ignore error await safe(s3.abortMultipartUpload({ Bucket: config.bucket, Key: multipartUpload.Key, UploadId: multipartUpload.UploadId }), { debug: log }); // ignore error
} }
} }
@@ -535,7 +535,7 @@ async function copyInternal(config, fullFromPath, fullToPath, fileSize, progress
const s3 = createS3Client(config, { retryStrategy: RETRY_STRATEGY }); // https://docs.aws.amazon.com/sdkref/latest/guide/feature-retry-behavior.html const s3 = createS3Client(config, { retryStrategy: RETRY_STRATEGY }); // https://docs.aws.amazon.com/sdkref/latest/guide/feature-retry-behavior.html
function throwError(error) { function throwError(error) {
if (error) debug(`copy: s3 copy error when copying ${fullFromPath}: ${error}`); if (error) log(`copy: s3 copy error when copying ${fullFromPath}: ${error}`);
if (error && S3_NOT_FOUND(error)) throw new BoxError(BoxError.NOT_FOUND, `Old backup not found: ${fullFromPath}`); if (error && S3_NOT_FOUND(error)) throw new BoxError(BoxError.NOT_FOUND, `Old backup not found: ${fullFromPath}`);
if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, `Error copying ${fullFromPath} (${fileSize} bytes): ${error.Code || ''} ${error}`); if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, `Error copying ${fullFromPath} (${fileSize} bytes): ${error.Code || ''} ${error}`);
@@ -603,7 +603,7 @@ async function copyInternal(config, fullFromPath, fullToPath, fileSize, progress
UploadId: uploadId UploadId: uploadId
}; };
progressCallback({ message: `Aborting multipart copy of ${fullFromPath}` }); progressCallback({ message: `Aborting multipart copy of ${fullFromPath}` });
await safe(s3.abortMultipartUpload(abortParams), { debug }); // ignore any abort errors await safe(s3.abortMultipartUpload(abortParams), { debug: log }); // ignore any abort errors
return throwError(copyError); return throwError(copyError);
} }

14
src/syncer.js
View File

@@ -1,14 +1,14 @@
import assert from 'node:assert'; import assert from 'node:assert';
import BoxError from './boxerror.js'; import BoxError from './boxerror.js';
import DataLayout from './datalayout.js'; import DataLayout from './datalayout.js';
import debugModule from 'debug'; import logger from './logger.js';
import fs from 'node:fs'; import fs from 'node:fs';
import path from 'node:path'; import path from 'node:path';
import readline from 'node:readline'; import readline from 'node:readline';
import safe from 'safetydance'; import safe from 'safetydance';
import util from 'node:util'; import util from 'node:util';
const debug = debugModule('box:syncer'); const { log, trace } = logger('syncer');
function readCache(cacheFile) { function readCache(cacheFile) {
@@ -72,13 +72,13 @@ async function sync(dataLayout, cacheFile) {
// if cache is missing or if we crashed/errored in previous run, start out empty // if cache is missing or if we crashed/errored in previous run, start out empty
if (!safe.fs.existsSync(cacheFile)) { if (!safe.fs.existsSync(cacheFile)) {
debug(`sync: cache file ${cacheFile} is missing, starting afresh`); log(`sync: cache file ${cacheFile} is missing, starting afresh`);
delQueue.push({ operation: 'removedir', path: '', reason: 'nocache' }); delQueue.push({ operation: 'removedir', path: '', reason: 'nocache' });
} else if (safe.fs.existsSync(newCacheFile)) { } else if (safe.fs.existsSync(newCacheFile)) {
debug(`sync: new cache file ${newCacheFile} exists. previous run crashed, starting afresh`); log(`sync: new cache file ${newCacheFile} exists. previous run crashed, starting afresh`);
delQueue.push({ operation: 'removedir', path: '', reason: 'crash' }); delQueue.push({ operation: 'removedir', path: '', reason: 'crash' });
} else { } else {
debug(`sync: loading cache file ${cacheFile}`); log(`sync: loading cache file ${cacheFile}`);
cache = readCache(cacheFile); cache = readCache(cacheFile);
} }
@@ -168,7 +168,7 @@ async function finalize(integrityMap, cacheFile) {
const newCacheFile = `${cacheFile}.new`, tempCacheFile = `${cacheFile}.tmp`; const newCacheFile = `${cacheFile}.new`, tempCacheFile = `${cacheFile}.tmp`;
debug(`finalize: patching in integrity information into ${cacheFile}`); log(`finalize: patching in integrity information into ${cacheFile}`);
const tempCacheFd = safe.fs.openSync(tempCacheFile, 'w'); // truncates any existing file const tempCacheFd = safe.fs.openSync(tempCacheFile, 'w'); // truncates any existing file
if (tempCacheFd === -1) throw new BoxError(BoxError.FS_ERROR, 'Error opening temp cache file: ' + safe.error.message); if (tempCacheFd === -1) throw new BoxError(BoxError.FS_ERROR, 'Error opening temp cache file: ' + safe.error.message);
@@ -192,7 +192,7 @@ async function finalize(integrityMap, cacheFile) {
safe.fs.unlinkSync(cacheFile); safe.fs.unlinkSync(cacheFile);
safe.fs.unlinkSync(newCacheFile); safe.fs.unlinkSync(newCacheFile);
if (!safe.fs.renameSync(tempCacheFile, cacheFile)) debug('Unable to save new cache file'); if (!safe.fs.renameSync(tempCacheFile, cacheFile)) log('Unable to save new cache file');
} }
export default { export default {

16
src/system.js
View File

@@ -4,7 +4,7 @@ import asynctask from './asynctask.js';
const { AsyncTask } = asynctask; const { AsyncTask } = asynctask;
import backupSites from './backupsites.js'; import backupSites from './backupsites.js';
import BoxError from './boxerror.js'; import BoxError from './boxerror.js';
import debugModule from 'debug'; import logger from './logger.js';
import df from './df.js'; import df from './df.js';
import docker from './docker.js'; import docker from './docker.js';
import eventlog from './eventlog.js'; import eventlog from './eventlog.js';
@@ -18,7 +18,7 @@ import safe from 'safetydance';
import shellModule from './shell.js'; import shellModule from './shell.js';
import volumes from './volumes.js'; import volumes from './volumes.js';
const debug = debugModule('box:system'); const { log, trace } = logger('system');
const shell = shellModule('system'); const shell = shellModule('system');
@@ -168,7 +168,7 @@ async function getFilesystems() {
} }
async function checkDiskSpace() { async function checkDiskSpace() {
debug('checkDiskSpace: checking disk space'); log('checkDiskSpace: checking disk space');
const filesystems = await getFilesystems(); const filesystems = await getFilesystems();
@@ -185,7 +185,7 @@ async function checkDiskSpace() {
} }
} }
debug(`checkDiskSpace: disk space checked. low disk space: ${markdownMessage || 'no'}`); log(`checkDiskSpace: disk space checked. low disk space: ${markdownMessage || 'no'}`);
if (markdownMessage) { if (markdownMessage) {
const finalMessage = `One or more file systems are running low on space. Please increase the disk size at the earliest.\n\n${markdownMessage}`; const finalMessage = `One or more file systems are running low on space. Please increase the disk size at the earliest.\n\n${markdownMessage}`;
@@ -223,7 +223,7 @@ class FilesystemUsageTask extends AsyncTask {
if (type === 'ext4' || type === 'xfs') { // hdparm only works with block devices if (type === 'ext4' || type === 'xfs') { // hdparm only works with block devices
this.emitProgress(percent, 'Calculating Disk Speed'); this.emitProgress(percent, 'Calculating Disk Speed');
const [speedError, speed] = await safe(hdparm(filesystem, { abortSignal })); const [speedError, speed] = await safe(hdparm(filesystem, { abortSignal }));
if (speedError) debug(`hdparm error ${filesystem}: ${speedError.message}`); if (speedError) log(`hdparm error ${filesystem}: ${speedError.message}`);
this.emitData({ speed: speedError ? -1 : speed }); this.emitData({ speed: speedError ? -1 : speed });
} else { } else {
this.emitData({ speed: -1 }); this.emitData({ speed: -1 });
@@ -241,7 +241,7 @@ class FilesystemUsageTask extends AsyncTask {
content.usage = content.id === 'docker' ? dockerDf.LayersSize : dockerDf.Volumes.map((v) => v.UsageData.Size).reduce((a,b) => a + b, 0); content.usage = content.id === 'docker' ? dockerDf.LayersSize : dockerDf.Volumes.map((v) => v.UsageData.Size).reduce((a,b) => a + b, 0);
} else { } else {
const [error, duResult] = await safe(du(content.path, { abortSignal })); const [error, duResult] = await safe(du(content.path, { abortSignal }));
if (error) debug(`du error ${content.path}: ${error.message}`); // can happen if app is installing etc if (error) log(`du error ${content.path}: ${error.message}`); // can happen if app is installing etc
content.usage = duResult || 0; content.usage = duResult || 0;
} }
usage += content.usage; usage += content.usage;
@@ -266,7 +266,7 @@ async function reboot() {
await notifications.unpin(notifications.TYPE_REBOOT, {}); await notifications.unpin(notifications.TYPE_REBOOT, {});
const [error] = await safe(shell.sudo([ REBOOT_CMD ], {})); const [error] = await safe(shell.sudo([ REBOOT_CMD ], {}));
if (error) debug('reboot: could not reboot. %o', error); if (error) log('reboot: could not reboot. %o', error);
} }
async function getInfo() { async function getInfo() {
@@ -369,7 +369,7 @@ async function checkUbuntuVersion() {
} }
async function runSystemChecks() { async function runSystemChecks() {
debug('runSystemChecks: checking status'); log('runSystemChecks: checking status');
const checks = [ const checks = [
checkRebootRequired(), checkRebootRequired(),

28
src/tasks.js
View File

@@ -1,7 +1,7 @@
import assert from 'node:assert'; import assert from 'node:assert';
import BoxError from './boxerror.js'; import BoxError from './boxerror.js';
import database from './database.js'; import database from './database.js';
import debugModule from 'debug'; import logger from './logger.js';
import logs from './logs.js'; import logs from './logs.js';
import mysql from 'mysql2'; import mysql from 'mysql2';
import path from 'node:path'; import path from 'node:path';
@@ -10,7 +10,7 @@ import safe from 'safetydance';
import shellModule from './shell.js'; import shellModule from './shell.js';
import _ from './underscore.js'; import _ from './underscore.js';
const debug = debugModule('box:tasks'); const { log, trace } = logger('tasks');
const shell = shellModule('tasks'); const shell = shellModule('tasks');
const ESTOPPED = 'stopped'; const ESTOPPED = 'stopped';
@@ -70,7 +70,7 @@ async function update(id, task) {
assert.strictEqual(typeof id, 'string'); assert.strictEqual(typeof id, 'string');
assert.strictEqual(typeof task, 'object'); assert.strictEqual(typeof task, 'object');
debug(`updating task ${id} with: ${JSON.stringify(task)}`); log(`updating task ${id} with: ${JSON.stringify(task)}`);
const args = [], fields = []; const args = [], fields = [];
for (const k in task) { for (const k in task) {
@@ -92,7 +92,7 @@ async function setCompleted(id, task) {
assert.strictEqual(typeof id, 'string'); assert.strictEqual(typeof id, 'string');
assert.strictEqual(typeof task, 'object'); assert.strictEqual(typeof task, 'object');
debug(`setCompleted - ${id}: ${JSON.stringify(task)}`); log(`setCompleted - ${id}: ${JSON.stringify(task)}`);
await update(id, Object.assign({ completed: true }, task)); await update(id, Object.assign({ completed: true }, task));
} }
@@ -145,7 +145,7 @@ async function stopTask(id) {
if (!gTasks[id]) throw new BoxError(BoxError.BAD_STATE, 'task is not active'); if (!gTasks[id]) throw new BoxError(BoxError.BAD_STATE, 'task is not active');
debug(`stopTask: stopping task ${id}`); log(`stopTask: stopping task ${id}`);
await shell.sudo([ STOP_TASK_CMD, id, ], {}); // note: this is stopping the systemd-run task. the sudo will exit when this exits await shell.sudo([ STOP_TASK_CMD, id, ], {}); // note: this is stopping the systemd-run task. the sudo will exit when this exits
} }
@@ -155,7 +155,7 @@ async function startTask(id, options) {
assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof options, 'object');
const logFile = options.logFile || `${paths.TASKS_LOG_DIR}/${id}.log`; const logFile = options.logFile || `${paths.TASKS_LOG_DIR}/${id}.log`;
debug(`startTask - starting task ${id} with options ${JSON.stringify(options)}. logs at ${logFile}`); log(`startTask - starting task ${id} with options ${JSON.stringify(options)}. logs at ${logFile}`);
const ac = new AbortController(); const ac = new AbortController();
gTasks[id] = ac; gTasks[id] = ac;
@@ -166,17 +166,17 @@ async function startTask(id, options) {
abortSignal: ac.signal, abortSignal: ac.signal,
timeout: options.timeout || 0, timeout: options.timeout || 0,
onTimeout: async () => { // custom stop because kill won't do. the task is running in some other process tree onTimeout: async () => { // custom stop because kill won't do. the task is running in some other process tree
debug(`onTimeout: ${id}`); log(`onTimeout: ${id}`);
await stopTask(id); await stopTask(id);
} }
}; };
safe(update(id, { pending: false }), { debug }); // background. we have to create the cp immediately to prevent race with stopTask() safe(update(id, { pending: false }), { debug: log }); // background. we have to create the cp immediately to prevent race with stopTask()
const [sudoError] = await safe(shell.sudo([ START_TASK_CMD, id, logFile, options.nice || 0, options.memoryLimit || 400, options.oomScoreAdjust || 0 ], sudoOptions)); const [sudoError] = await safe(shell.sudo([ START_TASK_CMD, id, logFile, options.nice || 0, options.memoryLimit || 400, options.oomScoreAdjust || 0 ], sudoOptions));
if (!gTasks[id]) { // when box code is shutting down, don't update the task status as "crashed". see stopAllTasks() if (!gTasks[id]) { // when box code is shutting down, don't update the task status as "crashed". see stopAllTasks()
debug(`startTask: ${id} completed as a result of box shutdown`); log(`startTask: ${id} completed as a result of box shutdown`);
return null; return null;
} }
@@ -186,7 +186,7 @@ async function startTask(id, options) {
if (!task) return null; // task disappeared on us. this can happen when db got cleared in tests if (!task) return null; // task disappeared on us. this can happen when db got cleared in tests
if (task.completed) { // task completed. we can trust the db result if (task.completed) { // task completed. we can trust the db result
debug(`startTask: ${id} completed. error: %o`, task.error); log(`startTask: ${id} completed. error: %o`, task.error);
if (task.error) throw task.error; if (task.error) throw task.error;
return task.result; return task.result;
} }
@@ -201,19 +201,19 @@ async function startTask(id, options) {
else if (sudoError.code === 50) taskError = { message:`Task ${id} crashed with code ${sudoError.code}`, code: ECRASHED }; else if (sudoError.code === 50) taskError = { message:`Task ${id} crashed with code ${sudoError.code}`, code: ECRASHED };
else taskError = { message:`Task ${id} crashed with unknown code ${sudoError.code}`, code: ECRASHED }; else taskError = { message:`Task ${id} crashed with unknown code ${sudoError.code}`, code: ECRASHED };
debug(`startTask: ${id} done. error: %o`, taskError); log(`startTask: ${id} done. error: %o`, taskError);
await safe(setCompleted(id, { error: taskError }), { debug }); await safe(setCompleted(id, { error: taskError }), { debug: log });
throw taskError; throw taskError;
} }
async function stopAllTasks() { async function stopAllTasks() {
const acs = Object.values(gTasks); const acs = Object.values(gTasks);
debug(`stopAllTasks: ${acs.length} tasks are running. sending abort signal`); log(`stopAllTasks: ${acs.length} tasks are running. sending abort signal`);
gTasks = {}; // this signals startTask() to not set completion status as "crashed" gTasks = {}; // this signals startTask() to not set completion status as "crashed"
acs.forEach(ac => ac.abort()); // cleanup all the sudos and systemd-run acs.forEach(ac => ac.abort()); // cleanup all the sudos and systemd-run
const [error] = await safe(shell.sudo([ STOP_TASK_CMD, 'all' ], { cwd: paths.baseDir() })); const [error] = await safe(shell.sudo([ STOP_TASK_CMD, 'all' ], { cwd: paths.baseDir() }));
if (error) debug(`stopAllTasks: error stopping stasks: ${error.message}`); if (error) log(`stopAllTasks: error stopping stasks: ${error.message}`);
} }
async function getLogs(task, options) { async function getLogs(task, options) {

16
src/taskworker.js
View File

@@ -19,7 +19,7 @@ import safe from 'safetydance';
import tasks from './tasks.js'; import tasks from './tasks.js';
import timers from 'timers/promises'; import timers from 'timers/promises';
import updater from './updater.js'; import updater from './updater.js';
import debugModule from 'debug'; import logger from './logger.js';
const TASKS = { // indexed by task type const TASKS = { // indexed by task type
app: apptask.run, app: apptask.run,
@@ -100,28 +100,28 @@ async function main() {
return process.exit(50); return process.exit(50);
} }
const debug = debugModule('box:taskworker'); // import this here so that logging handler is already setup const { log, trace } = logger('taskworker'); // import this here so that logging handler is already setup
process.on('SIGTERM', () => { process.on('SIGTERM', () => {
debug('Terminated'); log('Terminated');
exitSync({ code: 70 }); exitSync({ code: 70 });
}); });
// ensure we log task crashes with the task logs. neither console.log nor debug are sync for some reason // ensure we log task crashes with the task logs. neither console.log nor debug are sync for some reason
process.on('uncaughtException', (error) => exitSync({ error, code: 1 })); process.on('uncaughtException', (error) => exitSync({ error, code: 1 }));
debug(`Starting task ${taskId}. Logs are at ${logFile}`); log(`Starting task ${taskId}. Logs are at ${logFile}`);
const [getError, task] = await safe(tasks.get(taskId)); const [getError, task] = await safe(tasks.get(taskId));
if (getError) return exitSync({ error: getError, code: 50 }); if (getError) return exitSync({ error: getError, code: 50 });
if (!task) return exitSync({ error: new Error(`Task ${taskId} not found`), code: 50 }); if (!task) return exitSync({ error: new Error(`Task ${taskId} not found`), code: 50 });
async function progressCallback(progress) { async function progressCallback(progress) {
await safe(tasks.update(taskId, progress), { debug }); await safe(tasks.update(taskId, progress), { debug: log });
} }
const taskName = task.type.replace(/_.*/,''); const taskName = task.type.replace(/_.*/,'');
debug(`Running task of type ${taskName}`); log(`Running task of type ${taskName}`);
const [runError, result] = await safe(TASKS[taskName].apply(null, task.args.concat(progressCallback))); const [runError, result] = await safe(TASKS[taskName].apply(null, task.args.concat(progressCallback)));
const progress = { const progress = {
result: result || null, result: result || null,
@@ -129,9 +129,9 @@ async function main() {
percent: 100 percent: 100
}; };
await safe(tasks.setCompleted(taskId, progress), { debug }); await safe(tasks.setCompleted(taskId, progress), { debug: log });
debug(`Task took ${(new Date() - startTime)/1000} seconds`); log(`Task took ${(new Date() - startTime)/1000} seconds`);
exitSync({ error: runError, code: (!runError || runError instanceof BoxError) ? 0 : 50 }); // handled error vs run time crash exitSync({ error: runError, code: (!runError || runError instanceof BoxError) ? 0 : 50 }); // handled error vs run time crash
} }

10
src/translations.js
View File

@@ -1,12 +1,12 @@
import assert from 'node:assert'; import assert from 'node:assert';
import cloudron from './cloudron.js'; import cloudron from './cloudron.js';
import debugModule from 'debug'; import logger from './logger.js';
import fs from 'node:fs'; import fs from 'node:fs';
import path from 'node:path'; import path from 'node:path';
import paths from './paths.js'; import paths from './paths.js';
import safe from 'safetydance'; import safe from 'safetydance';
const debug = debugModule('box:translation'); const { log, trace } = logger('translation');
// to be used together with getTranslations() => { translations, fallback } // to be used together with getTranslations() => { translations, fallback }
@@ -45,13 +45,13 @@ function translate(input, assets) {
async function getTranslations() { async function getTranslations() {
const fallbackData = fs.readFileSync(path.join(paths.TRANSLATIONS_DIR, 'en.json'), 'utf8'); const fallbackData = fs.readFileSync(path.join(paths.TRANSLATIONS_DIR, 'en.json'), 'utf8');
if (!fallbackData) debug(`getTranslations: Fallback language en not found. ${safe.error.message}`); if (!fallbackData) log(`getTranslations: Fallback language en not found. ${safe.error.message}`);
const fallback = safe.JSON.parse(fallbackData) || {}; const fallback = safe.JSON.parse(fallbackData) || {};
const lang = await cloudron.getLanguage(); const lang = await cloudron.getLanguage();
const translationData = safe.fs.readFileSync(path.join(paths.TRANSLATIONS_DIR, `${lang}.json`), 'utf8'); const translationData = safe.fs.readFileSync(path.join(paths.TRANSLATIONS_DIR, `${lang}.json`), 'utf8');
if (!translationData) debug(`getTranslations: Requested language ${lang} not found. ${safe.error.message}`); if (!translationData) log(`getTranslations: Requested language ${lang} not found. ${safe.error.message}`);
const translations = safe.JSON.parse(translationData) || {}; const translations = safe.JSON.parse(translationData) || {};
return { translations, fallback }; return { translations, fallback };
@@ -60,7 +60,7 @@ async function getTranslations() {
async function listLanguages() { async function listLanguages() {
const [error, result] = await safe(fs.promises.readdir(paths.TRANSLATIONS_DIR)); const [error, result] = await safe(fs.promises.readdir(paths.TRANSLATIONS_DIR));
if (error) { if (error) {
debug(`listLanguages: Failed to list translations. %${error.message}`); log(`listLanguages: Failed to list translations. %${error.message}`);
return [ 'en' ]; // we always return english to avoid dashboard breakage return [ 'en' ]; // we always return english to avoid dashboard breakage
} }

48
src/updater.js
View File

@@ -10,7 +10,7 @@ import constants from './constants.js';
import cron from './cron.js'; import cron from './cron.js';
import { CronTime } from 'cron'; import { CronTime } from 'cron';
import crypto from 'node:crypto'; import crypto from 'node:crypto';
import debugModule from 'debug'; import logger from './logger.js';
import df from './df.js'; import df from './df.js';
import eventlog from './eventlog.js'; import eventlog from './eventlog.js';
import fs from 'node:fs'; import fs from 'node:fs';
@@ -26,7 +26,7 @@ import settings from './settings.js';
import shellModule from './shell.js'; import shellModule from './shell.js';
import tasks from './tasks.js'; import tasks from './tasks.js';
const debug = debugModule('box:updater'); const { log, trace } = logger('updater');
const shell = shellModule('updater'); const shell = shellModule('updater');
@@ -58,11 +58,11 @@ async function downloadBoxUrl(url, file) {
safe.fs.unlinkSync(file); safe.fs.unlinkSync(file);
await promiseRetry({ times: 10, interval: 5000, debug }, async function () { await promiseRetry({ times: 10, interval: 5000, debug: log }, async function () {
debug(`downloadBoxUrl: downloading ${url} to ${file}`); log(`downloadBoxUrl: downloading ${url} to ${file}`);
const [error] = await safe(shell.spawn('curl', ['-s', '--fail', url, '-o', file], { encoding: 'utf8' })); const [error] = await safe(shell.spawn('curl', ['-s', '--fail', url, '-o', file], { encoding: 'utf8' }));
if (error) throw new BoxError(BoxError.NETWORK_ERROR, `Failed to download ${url}: ${error.message}`); if (error) throw new BoxError(BoxError.NETWORK_ERROR, `Failed to download ${url}: ${error.message}`);
debug('downloadBoxUrl: done'); log('downloadBoxUrl: done');
}); });
} }
@@ -72,13 +72,13 @@ async function gpgVerifyBoxTarball(file, sig) {
const [error, stdout] = await safe(shell.spawn('/usr/bin/gpg', ['--status-fd', '1', '--no-default-keyring', '--keyring', RELEASES_PUBLIC_KEY, '--verify', sig, file], { encoding: 'utf8' })); const [error, stdout] = await safe(shell.spawn('/usr/bin/gpg', ['--status-fd', '1', '--no-default-keyring', '--keyring', RELEASES_PUBLIC_KEY, '--verify', sig, file], { encoding: 'utf8' }));
if (error) { if (error) {
debug(`gpgVerifyBoxTarball: command failed. error: ${error}\n stdout: ${error.stdout}\n stderr: ${error.stderr}`); log(`gpgVerifyBoxTarball: command failed. error: ${error}\n stdout: ${error.stdout}\n stderr: ${error.stderr}`);
throw new BoxError(BoxError.NOT_SIGNED, `The signature in ${path.basename(sig)} could not be verified (command failed)`); throw new BoxError(BoxError.NOT_SIGNED, `The signature in ${path.basename(sig)} could not be verified (command failed)`);
} }
if (stdout.indexOf('[GNUPG:] VALIDSIG 0EADB19CDDA23CD0FE71E3470A372F8703C493CC') !== -1) return; // success if (stdout.indexOf('[GNUPG:] VALIDSIG 0EADB19CDDA23CD0FE71E3470A372F8703C493CC') !== -1) return; // success
debug(`gpgVerifyBoxTarball: verification of ${sig} failed: ${stdout}\n`); log(`gpgVerifyBoxTarball: verification of ${sig} failed: ${stdout}\n`);
throw new BoxError(BoxError.NOT_SIGNED, `The signature in ${path.basename(sig)} could not be verified (bad sig)`); throw new BoxError(BoxError.NOT_SIGNED, `The signature in ${path.basename(sig)} could not be verified (bad sig)`);
} }
@@ -87,13 +87,13 @@ async function extractBoxTarball(tarball, dir) {
assert.strictEqual(typeof tarball, 'string'); assert.strictEqual(typeof tarball, 'string');
assert.strictEqual(typeof dir, 'string'); assert.strictEqual(typeof dir, 'string');
debug(`extractBoxTarball: extracting ${tarball} to ${dir}`); log(`extractBoxTarball: extracting ${tarball} to ${dir}`);
const [error] = await safe(shell.spawn('tar', ['-zxf', tarball, '-C', dir], { encoding: 'utf8' })); const [error] = await safe(shell.spawn('tar', ['-zxf', tarball, '-C', dir], { encoding: 'utf8' }));
if (error) throw new BoxError(BoxError.FS_ERROR, `Failed to extract release package: ${error.message}`); if (error) throw new BoxError(BoxError.FS_ERROR, `Failed to extract release package: ${error.message}`);
safe.fs.unlinkSync(tarball); safe.fs.unlinkSync(tarball);
debug('extractBoxTarball: extracted'); log('extractBoxTarball: extracted');
} }
async function verifyBoxUpdateInfo(versionsFile, updateInfo) { async function verifyBoxUpdateInfo(versionsFile, updateInfo) {
@@ -115,7 +115,7 @@ async function downloadAndVerifyBoxRelease(updateInfo) {
const oldArtifactNames = filenames.filter(f => f.startsWith('box-')); const oldArtifactNames = filenames.filter(f => f.startsWith('box-'));
for (const artifactName of oldArtifactNames) { for (const artifactName of oldArtifactNames) {
const fullPath = path.join(os.tmpdir(), artifactName); const fullPath = path.join(os.tmpdir(), artifactName);
debug(`downloadAndVerifyBoxRelease: removing old artifact ${fullPath}`); log(`downloadAndVerifyBoxRelease: removing old artifact ${fullPath}`);
await fs.promises.rm(fullPath, { recursive: true, force: true }); await fs.promises.rm(fullPath, { recursive: true, force: true });
} }
@@ -173,7 +173,7 @@ async function updateBox(boxUpdateInfo, options, progressCallback) {
await locks.wait(locks.TYPE_BOX_UPDATE); await locks.wait(locks.TYPE_BOX_UPDATE);
debug(`Updating box with ${boxUpdateInfo.sourceTarballUrl}`); log(`Updating box with ${boxUpdateInfo.sourceTarballUrl}`);
progressCallback({ percent: 70, message: 'Installing update...' }); progressCallback({ percent: 70, message: 'Installing update...' });
const [error] = await safe(shell.sudo([ UPDATE_CMD, packageInfo.file, process.stdout.logFile ], {})); // run installer.sh from new box code as a separate service const [error] = await safe(shell.sudo([ UPDATE_CMD, packageInfo.file, process.stdout.logFile ], {})); // run installer.sh from new box code as a separate service
if (error) { if (error) {
@@ -226,9 +226,9 @@ async function startBoxUpdateTask(options, auditSource) {
// background // background
tasks.startTask(taskId, { timeout: 20 * 60 * 60 * 1000 /* 20 hours */, nice: 15, memoryLimit }) tasks.startTask(taskId, { timeout: 20 * 60 * 60 * 1000 /* 20 hours */, nice: 15, memoryLimit })
.then(() => debug('startBoxUpdateTask: update task completed')) .then(() => log('startBoxUpdateTask: update task completed'))
.catch(async (updateError) => { .catch(async (updateError) => {
debug('Update failed with error. %o', updateError); log('Update failed with error. %o', updateError);
await locks.release(locks.TYPE_BOX_UPDATE_TASK); await locks.release(locks.TYPE_BOX_UPDATE_TASK);
await locks.releaseByTaskId(taskId); await locks.releaseByTaskId(taskId);
@@ -249,7 +249,7 @@ async function notifyBoxUpdate() {
if (!version) { if (!version) {
await eventlog.add(eventlog.ACTION_INSTALL_FINISH, AuditSource.CRON, { version: constants.VERSION }); await eventlog.add(eventlog.ACTION_INSTALL_FINISH, AuditSource.CRON, { version: constants.VERSION });
} else { } else {
debug(`notifyBoxUpdate: update finished from ${version} to ${constants.VERSION}`); log(`notifyBoxUpdate: update finished from ${version} to ${constants.VERSION}`);
const [error] = await safe(tasks.setCompletedByType(tasks.TASK_BOX_UPDATE, { error: null })); const [error] = await safe(tasks.setCompletedByType(tasks.TASK_BOX_UPDATE, { error: null }));
if (error && error.reason !== BoxError.NOT_FOUND) throw error; // when hotfixing, task may not exist if (error && error.reason !== BoxError.NOT_FOUND) throw error; // when hotfixing, task may not exist
await eventlog.add(eventlog.ACTION_UPDATE_FINISH, AuditSource.CRON, { errorMessage: '', oldVersion: version || 'dev', newVersion: constants.VERSION }); await eventlog.add(eventlog.ACTION_UPDATE_FINISH, AuditSource.CRON, { errorMessage: '', oldVersion: version || 'dev', newVersion: constants.VERSION });
@@ -265,10 +265,10 @@ async function autoUpdate(auditSource) {
const boxUpdateInfo = await getBoxUpdate(); const boxUpdateInfo = await getBoxUpdate();
// do box before app updates. for the off chance that the box logic fixes some app update logic issue // do box before app updates. for the off chance that the box logic fixes some app update logic issue
if (boxUpdateInfo && !boxUpdateInfo.unstable) { if (boxUpdateInfo && !boxUpdateInfo.unstable) {
debug('autoUpdate: starting box autoupdate to %j', boxUpdateInfo.version); log('autoUpdate: starting box autoupdate to %j', boxUpdateInfo.version);
const [error] = await safe(startBoxUpdateTask({ skipBackup: false }, AuditSource.CRON)); const [error] = await safe(startBoxUpdateTask({ skipBackup: false }, AuditSource.CRON));
if (!error) return; // do not start app updates when a box update got scheduled if (!error) return; // do not start app updates when a box update got scheduled
debug(`autoUpdate: failed to start box autoupdate task: ${error.message}`); log(`autoUpdate: failed to start box autoupdate task: ${error.message}`);
// fall through to update apps if box update never started (failed ubuntu or avx check) // fall through to update apps if box update never started (failed ubuntu or avx check)
} }
@@ -276,13 +276,13 @@ async function autoUpdate(auditSource) {
for (const app of result) { for (const app of result) {
if (!app.updateInfo) continue; if (!app.updateInfo) continue;
if (!app.updateInfo.isAutoUpdatable) { if (!app.updateInfo.isAutoUpdatable) {
debug(`autoUpdate: ${app.fqdn} requires manual update. skipping`); log(`autoUpdate: ${app.fqdn} requires manual update. skipping`);
continue; continue;
} }
const sites = await backupSites.listByContentForUpdates(app.id); const sites = await backupSites.listByContentForUpdates(app.id);
if (sites.length === 0) { if (sites.length === 0) {
debug(`autoUpdate: ${app.fqdn} has no backup site for updates. skipping`); log(`autoUpdate: ${app.fqdn} has no backup site for updates. skipping`);
continue; continue;
} }
@@ -291,9 +291,9 @@ async function autoUpdate(auditSource) {
force: false force: false
}; };
debug(`autoUpdate: ${app.fqdn} will be automatically updated`); log(`autoUpdate: ${app.fqdn} will be automatically updated`);
const [updateError] = await safe(apps.updateApp(app, data, auditSource)); const [updateError] = await safe(apps.updateApp(app, data, auditSource));
if (updateError) debug(`autoUpdate: error autoupdating ${app.fqdn}: ${updateError.message}`); if (updateError) log(`autoUpdate: error autoupdating ${app.fqdn}: ${updateError.message}`);
} }
} }
@@ -320,7 +320,7 @@ async function checkAppUpdate(app, options) {
async function checkBoxUpdate(options) { async function checkBoxUpdate(options) {
assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof options, 'object');
debug('checkBoxUpdate: checking for updates'); log('checkBoxUpdate: checking for updates');
const updateInfo = await appstore.getBoxUpdate(options); const updateInfo = await appstore.getBoxUpdate(options);
if (updateInfo) { if (updateInfo) {
@@ -346,7 +346,7 @@ async function raiseNotifications() {
// currently, we do not raise notifications when auto-update is disabled. separate notifications appears spammy when having many apps // currently, we do not raise notifications when auto-update is disabled. separate notifications appears spammy when having many apps
// in the future, we can maybe aggregate? // in the future, we can maybe aggregate?
if (app.updateInfo && !app.updateInfo.isAutoUpdatable) { if (app.updateInfo && !app.updateInfo.isAutoUpdatable) {
debug(`autoUpdate: ${app.fqdn} cannot be autoupdated. skipping`); log(`autoUpdate: ${app.fqdn} cannot be autoupdated. skipping`);
await notifications.pin(notifications.TYPE_MANUAL_APP_UPDATE_NEEDED, `${app.manifest.title} at ${app.fqdn} requires manual update to version ${app.updateInfo.manifest.version}`, await notifications.pin(notifications.TYPE_MANUAL_APP_UPDATE_NEEDED, `${app.manifest.title} at ${app.fqdn} requires manual update to version ${app.updateInfo.manifest.version}`,
`Changelog:\n${app.updateInfo.manifest.changelog}\n`, { context: app.id }); `Changelog:\n${app.updateInfo.manifest.changelog}\n`, { context: app.id });
continue; continue;
@@ -358,12 +358,12 @@ async function checkForUpdates(options) {
assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof options, 'object');
const [boxError] = await safe(checkBoxUpdate(options)); const [boxError] = await safe(checkBoxUpdate(options));
if (boxError) debug('checkForUpdates: error checking for box updates: %o', boxError); if (boxError) log('checkForUpdates: error checking for box updates: %o', boxError);
// check app updates // check app updates
const result = await apps.list(); const result = await apps.list();
for (const app of result) { for (const app of result) {
await safe(checkAppUpdate(app, options), { debug }); await safe(checkAppUpdate(app, options), { debug: log });
} }
// raise notifications here because the updatechecker runs regardless of auto-updater cron job // raise notifications here because the updatechecker runs regardless of auto-updater cron job

Some files were not shown because too many files have changed in this diff Show More