jjkiers/cloudron-box
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

replace debug() with our custom logger

Browse Source 
mostly we want trace() and log(). trace() can be enabled whenever
we want by flipping a flag and restarting box
This commit is contained in:
Girish Ramakrishnan
2026-03-12 22:55:28 +05:30
parent d57554a48c
commit 01d0c738bc
104 changed files with 1187 additions and 1174 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
box.js
View File

@@ -10,9 +10,9 @@ import proxyAuth from './src/proxyauth.js';
import safe from 'safetydance';
import server from './src/server.js';
import directoryServer from './src/directoryserver.js';
import debugModule from 'debug';
import logger from './src/logger.js';
const debug = debugModule('box:box');
const { log, trace } = logger('box');
let logFd;
@@ -59,13 +59,13 @@ const [error] = await safe(startServers());
if (error) exitSync({ error, code: 1, message: 'Error starting servers' });
process.on('SIGHUP', async function () {
debug('Received SIGHUP. Re-reading configs.');
log('Received SIGHUP. Re-reading configs.');
const conf = await directoryServer.getConfig();
if (conf.enabled) await directoryServer.checkCertificate();
});
process.on('SIGINT', async function () {
debug('Received SIGINT. Shutting down.');
log('Received SIGINT. Shutting down.');
await proxyAuth.stop();
await server.stop();
@@ -74,13 +74,13 @@ process.on('SIGINT', async function () {
await oidcServer.stop();
setTimeout(() => {
debug('Shutdown complete');
log('Shutdown complete');
process.exit();
}, 2000); // need to wait for the task processes to die
});
process.on('SIGTERM', async function () {
debug('Received SIGTERM. Shutting down.');
log('Received SIGTERM. Shutting down.');
await proxyAuth.stop();
await server.stop();
@@ -89,7 +89,7 @@ process.on('SIGTERM', async function () {
await oidcServer.stop();
setTimeout(() => {
debug('Shutdown complete');
log('Shutdown complete');
process.exit();
}, 2000); // need to wait for the task processes to die
});

1
package-lock.json generated
View File

@@ -29,7 +29,6 @@
"cron": "^4.4.0",
"db-migrate": "^0.11.14",
"db-migrate-mysql": "^3.0.0",
"debug": "^4.4.3",
"dockerode": "^4.0.9",
"domrobot-client": "^3.3.0",
"ejs": "^4.0.1",

1
package.json
View File

@@ -34,7 +34,6 @@
"cron": "^4.4.0",
"db-migrate": "^0.11.14",
"db-migrate-mysql": "^3.0.0",
"debug": "^4.4.3",
"dockerode": "^4.0.9",
"domrobot-client": "^3.3.0",
"ejs": "^4.0.1",

3
setup/start/systemd/box.service
View File

@@ -16,8 +16,7 @@ Restart=always
ExecStart=/home/yellowtent/box/box.js
ExecReload=/bin/kill -HUP $MAINPID
; we run commands like df which will parse properly only with correct locale
; add "oidc-provider:*" to DEBUG for OpenID debugging
Environment="HOME=/home/yellowtent" "USER=yellowtent" "DEBUG=box:*,-box:ldapserver,-box:directoryserver,-box:oidcserver" "BOX_ENV=cloudron" "NODE_ENV=production" "LC_ALL=C"
Environment="HOME=/home/yellowtent" "USER=yellowtent" "BOX_ENV=cloudron" "NODE_ENV=production" "LC_ALL=C"
; this sends the main process SIGTERM and then if anything lingers to the control-group . this is also the case if the main process crashes.
; the box code handles SIGTERM and cleanups the tasks
KillMode=mixed

2
setup/start/systemd/cloudron-syslog.service
View File

@@ -5,7 +5,7 @@ After=network.target
[Service]
ExecStart=/home/yellowtent/box/syslog.js
WorkingDirectory=/home/yellowtent/box
Environment="NODE_ENV=production" "DEBUG=syslog:*" "BOX_ENV=cloudron"
Environment="NODE_ENV=production" "BOX_ENV=cloudron"
Restart=always
User=yellowtent
Group=yellowtent

84
src/acme2.js
View File

@@ -2,7 +2,7 @@ import assert from 'node:assert';
import blobs from './blobs.js';
import BoxError from './boxerror.js';
import crypto from 'node:crypto';
import debugModule from 'debug';
import logger from './logger.js';
import dns from './dns.js';
import openssl from './openssl.js';
import path from 'node:path';
@@ -12,7 +12,7 @@ import safe from 'safetydance';
import superagent from '@cloudron/superagent';
import users from './users.js';
const debug = debugModule('box:cert/acme2');
const { log, trace } = logger('cert/acme2');
const CA_PROD_DIRECTORY_URL = 'https://acme-v02.api.letsencrypt.org/directory',
CA_STAGING_DIRECTORY_URL = 'https://acme-staging-v02.api.letsencrypt.org/directory';
@@ -53,7 +53,7 @@ function Acme2(fqdn, domainObject, email, key, options) {
this.profile = options.profile || ''; // https://letsencrypt.org/docs/profiles/ . is validated against the directory
debug(`Acme2: will get cert for fqdn: ${this.fqdn} cn: ${this.cn} certName: ${this.certName} wildcard: ${this.wildcard} http: ${this.forceHttpAuthorization}`);
log(`Acme2: will get cert for fqdn: ${this.fqdn} cn: ${this.cn} certName: ${this.certName} wildcard: ${this.wildcard} http: ${this.forceHttpAuthorization}`);
}
// urlsafe base64 encoding (jose)
@@ -140,7 +140,7 @@ Acme2.prototype.sendSignedRequest = async function (url, payload) {
const nonce = response.headers['Replay-Nonce'.toLowerCase()];
if (!nonce) throw new BoxError(BoxError.ACME_ERROR, 'No nonce in response');
debug(`sendSignedRequest: using nonce ${nonce} for url ${url}`);
log(`sendSignedRequest: using nonce ${nonce} for url ${url}`);
const protected64 = b64(JSON.stringify(Object.assign({}, header, { nonce: nonce })));
@@ -168,7 +168,7 @@ Acme2.prototype.postAsGet = async function (url) {
Acme2.prototype.updateContact = async function (registrationUri) {
assert.strictEqual(typeof registrationUri, 'string');
debug(`updateContact: registrationUri: ${registrationUri} email: ${this.email}`);
log(`updateContact: registrationUri: ${registrationUri} email: ${this.email}`);
// https://github.com/ietf-wg-acme/acme/issues/30
const payload = {
@@ -178,7 +178,7 @@ Acme2.prototype.updateContact = async function (registrationUri) {
const result = await this.sendSignedRequest(registrationUri, JSON.stringify(payload));
if (result.status !== 200) throw new BoxError(BoxError.ACME_ERROR, `Failed to update contact. Expecting 200, got ${result.status} ${result.text}`);
debug(`updateContact: contact of user updated to ${this.email}`);
log(`updateContact: contact of user updated to ${this.email}`);
};
Acme2.prototype.ensureAccount = async function () {
@@ -186,18 +186,18 @@ Acme2.prototype.ensureAccount = async function () {
termsOfServiceAgreed: true
};
debug('ensureAccount: registering user');
log('ensureAccount: registering user');
this.accountKey = await blobs.getString(blobs.ACME_ACCOUNT_KEY);
if (!this.accountKey) {
debug('ensureAccount: generating new account keys');
log('ensureAccount: generating new account keys');
this.accountKey = await openssl.generateKey('rsa4096');
await blobs.setString(blobs.ACME_ACCOUNT_KEY, this.accountKey);
}
let result = await this.sendSignedRequest(this.directory.newAccount, JSON.stringify(payload));
if (result.status === 403 && result.body.type === 'urn:ietf:params:acme:error:unauthorized') {
debug(`ensureAccount: key was revoked. ${result.status} ${result.text}. generating new account key`);
log(`ensureAccount: key was revoked. ${result.status} ${result.text}. generating new account key`);
this.accountKey = await openssl.generateKey('rsa4096');
await blobs.setString(blobs.ACME_ACCOUNT_KEY, this.accountKey);
result = await this.sendSignedRequest(this.directory.newAccount, JSON.stringify(payload));
@@ -206,7 +206,7 @@ Acme2.prototype.ensureAccount = async function () {
// 200 if already exists. 201 for new accounts
if (result.status !== 200 && result.status !== 201) throw new BoxError(BoxError.ACME_ERROR, `Failed to register new account. Expecting 200 or 201, got ${result.status} ${result.text}`);
debug(`ensureAccount: user registered keyid: ${result.headers.location}`);
log(`ensureAccount: user registered keyid: ${result.headers.location}`);
this.accountKeyId = result.headers.location;
@@ -223,14 +223,14 @@ Acme2.prototype.newOrder = async function () {
});
});
debug(`newOrder: ${JSON.stringify(this.altNames)}`);
log(`newOrder: ${JSON.stringify(this.altNames)}`);
const result = await this.sendSignedRequest(this.directory.newOrder, JSON.stringify(payload));
if (result.status === 403) throw new BoxError(BoxError.ACCESS_DENIED, `Forbidden sending new order: ${result.body.detail}`);
if (result.status !== 201) throw new BoxError(BoxError.ACME_ERROR, `Failed to send new order. Expecting 201, got ${result.status} ${result.text}`);
const order = result.body, orderUrl = result.headers.location;
debug(`newOrder: created order ${this.cn} order: ${result.text} orderUrl: ${orderUrl}`);
log(`newOrder: created order ${this.cn} order: ${result.text} orderUrl: ${orderUrl}`);
if (!Array.isArray(order.authorizations)) throw new BoxError(BoxError.ACME_ERROR, 'invalid authorizations in order');
if (typeof order.finalize !== 'string') throw new BoxError(BoxError.ACME_ERROR, 'invalid finalize in order');
@@ -242,18 +242,18 @@ Acme2.prototype.newOrder = async function () {
Acme2.prototype.waitForOrder = async function (orderUrl) {
assert.strictEqual(typeof orderUrl, 'string');
debug(`waitForOrder: ${orderUrl}`);
log(`waitForOrder: ${orderUrl}`);
return await promiseRetry({ times: 15, interval: 20000, debug }, async () => {
debug('waitForOrder: getting status');
return await promiseRetry({ times: 15, interval: 20000, debug: log }, async () => {
log('waitForOrder: getting status');
const result = await this.postAsGet(orderUrl);
if (result.status !== 200) {
debug(`waitForOrder: invalid response code getting uri ${result.status}`);
log(`waitForOrder: invalid response code getting uri ${result.status}`);
throw new BoxError(BoxError.ACME_ERROR, `Bad response when waiting for order. code: ${result.status}`);
}
debug('waitForOrder: status is "%s %j', result.body.status, result.body);
log('waitForOrder: status is "%s %j', result.body.status, result.body);
if (result.body.status === 'pending' || result.body.status === 'processing') throw new BoxError(BoxError.ACME_ERROR, `Request is in ${result.body.status} state`);
else if (result.body.status === 'valid' && result.body.certificate) return result.body.certificate;
@@ -279,7 +279,7 @@ Acme2.prototype.getKeyAuthorization = async function (token) {
Acme2.prototype.notifyChallengeReady = async function (challenge) {
assert.strictEqual(typeof challenge, 'object'); // { type, status, url, token }
debug(`notifyChallengeReady: ${challenge.url} was met`);
log(`notifyChallengeReady: ${challenge.url} was met`);
const keyAuthorization = await this.getKeyAuthorization(challenge.token);
@@ -295,18 +295,18 @@ Acme2.prototype.notifyChallengeReady = async function (challenge) {
Acme2.prototype.waitForChallenge = async function (challenge) {
assert.strictEqual(typeof challenge, 'object');
debug(`waitingForChallenge: ${JSON.stringify(challenge)}`);
log(`waitingForChallenge: ${JSON.stringify(challenge)}`);
await promiseRetry({ times: 15, interval: 20000, debug }, async () => {
debug('waitingForChallenge: getting status');
await promiseRetry({ times: 15, interval: 20000, debug: log }, async () => {
log('waitingForChallenge: getting status');
const result = await this.postAsGet(challenge.url);
if (result.status !== 200) {
debug(`waitForChallenge: invalid response code getting uri ${result.status}`);
log(`waitForChallenge: invalid response code getting uri ${result.status}`);
throw new BoxError(BoxError.ACME_ERROR, `Bad response code when waiting for challenge : ${result.status}`);
}
debug(`waitForChallenge: status is "${result.body.status}" "${result.text}"`);
log(`waitForChallenge: status is "${result.body.status}" "${result.text}"`);
if (result.body.status === 'pending') throw new BoxError(BoxError.ACME_ERROR, 'Challenge is in pending state');
else if (result.body.status === 'valid') return;
@@ -325,7 +325,7 @@ Acme2.prototype.signCertificate = async function (finalizationUrl, csrPem) {
csr: b64(csrDer)
};
debug(`signCertificate: sending sign request to ${finalizationUrl}`);
log(`signCertificate: sending sign request to ${finalizationUrl}`);
const result = await this.sendSignedRequest(finalizationUrl, JSON.stringify(payload));
// 429 means we reached the cert limit for this domain
@@ -335,8 +335,8 @@ Acme2.prototype.signCertificate = async function (finalizationUrl, csrPem) {
Acme2.prototype.downloadCertificate = async function (certUrl) {
assert.strictEqual(typeof certUrl, 'string');
return await promiseRetry({ times: 5, interval: 20000, debug }, async () => {
debug(`downloadCertificate: downloading certificate of ${this.cn}`);
return await promiseRetry({ times: 5, interval: 20000, debug: log }, async () => {
log(`downloadCertificate: downloading certificate of ${this.cn}`);
const result = await this.postAsGet(certUrl);
if (result.status === 202) throw new BoxError(BoxError.ACME_ERROR, 'Retry downloading certificate');
@@ -350,12 +350,12 @@ Acme2.prototype.downloadCertificate = async function (certUrl) {
Acme2.prototype.prepareHttpChallenge = async function (challenge) {
assert.strictEqual(typeof challenge, 'object');
debug(`prepareHttpChallenge: preparing for challenge ${JSON.stringify(challenge)}`);
log(`prepareHttpChallenge: preparing for challenge ${JSON.stringify(challenge)}`);
const keyAuthorization = await this.getKeyAuthorization(challenge.token);
const challengeFilePath = path.join(paths.ACME_CHALLENGES_DIR, challenge.token);
debug(`prepareHttpChallenge: writing ${keyAuthorization} to ${challengeFilePath}`);
log(`prepareHttpChallenge: writing ${keyAuthorization} to ${challengeFilePath}`);
if (!safe.fs.writeFileSync(challengeFilePath, keyAuthorization)) throw new BoxError(BoxError.FS_ERROR, `Error writing challenge: ${safe.error.message}`);
};
@@ -364,7 +364,7 @@ Acme2.prototype.cleanupHttpChallenge = async function (challenge) {
assert.strictEqual(typeof challenge, 'object');
const challengeFilePath = path.join(paths.ACME_CHALLENGES_DIR, challenge.token);
debug(`cleanupHttpChallenge: unlinking ${challengeFilePath}`);
log(`cleanupHttpChallenge: unlinking ${challengeFilePath}`);
if (!safe.fs.unlinkSync(challengeFilePath)) throw new BoxError(BoxError.FS_ERROR, `Error unlinking challenge: ${safe.error.message}`);
};
@@ -381,7 +381,7 @@ function getChallengeSubdomain(cn, domain) {
challengeSubdomain = '_acme-challenge.' + cn.slice(0, -domain.length - 1);
}
debug(`getChallengeSubdomain: challenge subdomain for cn ${cn} at domain ${domain} is ${challengeSubdomain}`);
log(`getChallengeSubdomain: challenge subdomain for cn ${cn} at domain ${domain} is ${challengeSubdomain}`);
return challengeSubdomain;
}
@@ -389,7 +389,7 @@ function getChallengeSubdomain(cn, domain) {
Acme2.prototype.prepareDnsChallenge = async function (cn, challenge) {
assert.strictEqual(typeof challenge, 'object');
debug(`prepareDnsChallenge: preparing for challenge: ${JSON.stringify(challenge)}`);
log(`prepareDnsChallenge: preparing for challenge: ${JSON.stringify(challenge)}`);
const keyAuthorization = await this.getKeyAuthorization(challenge.token);
const shasum = crypto.createHash('sha256');
@@ -398,7 +398,7 @@ Acme2.prototype.prepareDnsChallenge = async function (cn, challenge) {
const txtValue = urlBase64Encode(shasum.digest('base64'));
const challengeSubdomain = getChallengeSubdomain(cn, this.domain);
debug(`prepareDnsChallenge: update ${challengeSubdomain} with ${txtValue}`);
log(`prepareDnsChallenge: update ${challengeSubdomain} with ${txtValue}`);
await dns.upsertDnsRecords(challengeSubdomain, this.domain, 'TXT', [ `"${txtValue}"` ]);
@@ -416,7 +416,7 @@ Acme2.prototype.cleanupDnsChallenge = async function (cn, challenge) {
const txtValue = urlBase64Encode(shasum.digest('base64'));
const challengeSubdomain = getChallengeSubdomain(cn, this.domain);
debug(`cleanupDnsChallenge: remove ${challengeSubdomain} with ${txtValue}`);
log(`cleanupDnsChallenge: remove ${challengeSubdomain} with ${txtValue}`);
await dns.removeDnsRecords(challengeSubdomain, this.domain, 'TXT', [ `"${txtValue}"` ]);
};
@@ -425,7 +425,7 @@ Acme2.prototype.prepareChallenge = async function (cn, authorization) {
assert.strictEqual(typeof cn, 'string');
assert.strictEqual(typeof authorization, 'object');
debug(`prepareChallenge: http: ${this.forceHttpAuthorization} cn: ${cn} authorization: ${JSON.stringify(authorization)}`);
log(`prepareChallenge: http: ${this.forceHttpAuthorization} cn: ${cn} authorization: ${JSON.stringify(authorization)}`);
// validation is cached by LE for 60 days or so. if a user switches from non-wildcard DNS (http challenge) to programmatic DNS (dns challenge), then
// LE remembers the challenge type and won't give us a dns challenge for 60 days!
@@ -447,7 +447,7 @@ Acme2.prototype.cleanupChallenge = async function (cn, challenge) {
assert.strictEqual(typeof cn, 'string');
assert.strictEqual(typeof challenge, 'object');
debug(`cleanupChallenge: http: ${this.forceHttpAuthorization}`);
log(`cleanupChallenge: http: ${this.forceHttpAuthorization}`);
if (this.forceHttpAuthorization) {
await this.cleanupHttpChallenge(challenge);
@@ -461,7 +461,7 @@ Acme2.prototype.acmeFlow = async function () {
const { order, orderUrl } = await this.newOrder();
for (const authorizationUrl of order.authorizations) {
debug(`acmeFlow: authorizing ${authorizationUrl}`);
log(`acmeFlow: authorizing ${authorizationUrl}`);
const response = await this.postAsGet(authorizationUrl);
if (response.status !== 200) throw new BoxError(BoxError.ACME_ERROR, `Invalid response code getting authorization : ${response.status}`);
@@ -471,7 +471,7 @@ Acme2.prototype.acmeFlow = async function () {
const challenge = await this.prepareChallenge(cn, authorization);
await this.notifyChallengeReady(challenge);
await this.waitForChallenge(challenge);
await safe(this.cleanupChallenge(cn, challenge), { debug });
await safe(this.cleanupChallenge(cn, challenge), { debug: log });
}
const csr = await openssl.createCsr(this.key, this.cn, this.altNames);
@@ -485,7 +485,7 @@ Acme2.prototype.acmeFlow = async function () {
};
Acme2.prototype.loadDirectory = async function () {
await promiseRetry({ times: 3, interval: 20000, debug }, async () => {
await promiseRetry({ times: 3, interval: 20000, debug: log }, async () => {
const response = await superagent.get(this.caDirectory).timeout(30000).ok(() => true);
if (response.status !== 200) throw new BoxError(BoxError.ACME_ERROR, `Invalid response code when fetching directory : ${response.status}`);
@@ -502,11 +502,11 @@ Acme2.prototype.loadDirectory = async function () {
};
Acme2.prototype.getCertificate = async function () {
debug(`getCertificate: start acme flow for ${this.cn} from ${this.caDirectory}`);
log(`getCertificate: start acme flow for ${this.cn} from ${this.caDirectory}`);
await this.loadDirectory();
const result = await this.acmeFlow(); // { key, cert, csr, renewalInfo }
debug(`getCertificate: acme flow completed for ${this.cn}. renewalInfo: ${JSON.stringify(result.renewalInfo)}`);
log(`getCertificate: acme flow completed for ${this.cn}. renewalInfo: ${JSON.stringify(result.renewalInfo)}`);
return result;
};
@@ -522,8 +522,8 @@ async function getCertificate(fqdn, domainObject, key) {
const owner = await users.getOwner();
const email = owner?.email || 'webmaster@cloudron.io'; // can error if not activated yet
return await promiseRetry({ times: 3, interval: 0, debug }, async function () {
debug(`getCertificate: for fqdn ${fqdn} and domain ${domainObject.domain}`);
return await promiseRetry({ times: 3, interval: 0, debug: log }, async function () {
log(`getCertificate: for fqdn ${fqdn} and domain ${domainObject.domain}`);
const acme = new Acme2(fqdn, domainObject, email, key, { /* profile: 'shortlived' */ });
return await acme.getCertificate();

20
src/apphealthmonitor.js
View File

@@ -3,13 +3,13 @@ import assert from 'node:assert';
import AuditSource from './auditsource.js';
import BoxError from './boxerror.js';
import constants from './constants.js';
import debugModule from 'debug';
import logger from './logger.js';
import docker from './docker.js';
import eventlog from './eventlog.js';
import safe from 'safetydance';
import superagent from '@cloudron/superagent';
const debug = debugModule('box:apphealthmonitor');
const { log, trace } = logger('apphealthmonitor');
const UNHEALTHY_THRESHOLD = 20 * 60 * 1000; // 20 minutes
@@ -32,20 +32,20 @@ async function setHealth(app, health) {
if (health === apps.HEALTH_HEALTHY) {
healthTime = now;
if (lastHealth && lastHealth !== apps.HEALTH_HEALTHY) { // app starts out with null health
debug(`setHealth: ${app.id} (${app.fqdn}) switched from ${lastHealth} to healthy`);
log(`setHealth: ${app.id} (${app.fqdn}) switched from ${lastHealth} to healthy`);
// do not send mails for dev apps
if (!app.debugMode) await eventlog.add(eventlog.ACTION_APP_UP, AuditSource.HEALTH_MONITOR, { app });
}
} else if (Math.abs(now - healthTime) > UNHEALTHY_THRESHOLD) {
if (lastHealth === apps.HEALTH_HEALTHY) {
debug(`setHealth: marking ${app.id} (${app.fqdn}) as unhealthy since not seen for more than ${UNHEALTHY_THRESHOLD/(60 * 1000)} minutes`);
log(`setHealth: marking ${app.id} (${app.fqdn}) as unhealthy since not seen for more than ${UNHEALTHY_THRESHOLD/(60 * 1000)} minutes`);
// do not send mails for dev apps
if (!app.debugMode) await eventlog.add(eventlog.ACTION_APP_DOWN, AuditSource.HEALTH_MONITOR, { app });
}
} else {
debug(`setHealth: ${app.id} (${app.fqdn}) waiting for ${(UNHEALTHY_THRESHOLD - Math.abs(now - healthTime))/1000} to update health`);
log(`setHealth: ${app.id} (${app.fqdn}) waiting for ${(UNHEALTHY_THRESHOLD - Math.abs(now - healthTime))/1000} to update health`);
return;
}
@@ -137,7 +137,7 @@ async function processDockerEvents(options) {
const now = Date.now();
const notifyUser = !info?.app?.debugMode && ((now - gLastOomMailTime) > OOM_EVENT_LIMIT);
debug(`OOM ${program} notifyUser: ${notifyUser}. lastOomTime: ${gLastOomMailTime} (now: ${now})`);
log(`OOM ${program} notifyUser: ${notifyUser}. lastOomTime: ${gLastOomMailTime} (now: ${now})`);
if (notifyUser) {
await eventlog.add(eventlog.ACTION_APP_OOM, AuditSource.HEALTH_MONITOR, { event, containerId, addonName: info?.addonName || null, app: info?.app || null });
@@ -147,11 +147,11 @@ async function processDockerEvents(options) {
});
stream.on('error', function (error) {
debug('Error reading docker events: %o', error);
log('Error reading docker events: %o', error);
});
stream.on('end', function () {
// debug('Event stream ended');
// log('Event stream ended');
});
// safety hatch if 'until' doesn't work (there are cases where docker is working with a different time)
@@ -167,12 +167,12 @@ async function processApp(options) {
const results = await Promise.allSettled(healthChecks); // wait for all promises to finish
const unfulfilled = results.filter(r => r.status === 'rejected');
if (unfulfilled.length) debug(`app health: ${unfulfilled.length} health checks exceptions. e.g. ${unfulfilled[0].reason}`); // this should not happen
if (unfulfilled.length) log(`app health: ${unfulfilled.length} health checks exceptions. e.g. ${unfulfilled[0].reason}`); // this should not happen
const stopped = allApps.filter(app => app.runState === apps.RSTATE_STOPPED);
const running = allApps.filter(function (a) { return a.installationState === apps.ISTATE_INSTALLED && a.runState === apps.RSTATE_RUNNING && a.health === apps.HEALTH_HEALTHY; });
debug(`app health: ${running.length} running / ${stopped.length} stopped / ${allApps.length - running.length - stopped.length} unresponsive`);
log(`app health: ${running.length} running / ${stopped.length} stopped / ${allApps.length - running.length - stopped.length} unresponsive`);
}
async function run(intervalSecs) {

20
src/applinks.js
View File

@@ -3,12 +3,12 @@ import apps from './apps.js';
import BoxError from './boxerror.js';
import crypto from 'node:crypto';
import database from './database.js';
import debugModule from 'debug';
import logger from './logger.js';
import jsdom from 'jsdom';
import safe from 'safetydance';
import superagent from '@cloudron/superagent';
const debug = debugModule('box:applinks');
const { log, trace } = logger('applinks');
const APPLINKS_FIELDS= [ 'id', 'accessRestrictionJson', 'creationTime', 'updateTime', 'ts', 'label', 'tagsJson', 'icon', 'upstreamUri' ].join(',');
@@ -84,7 +84,7 @@ async function detectMetaInfo(upstreamUri) {
const [upstreamError, upstreamRespose] = await safe(superagent.get(upstreamUri).set('User-Agent', 'Mozilla').timeout(10*1000));
if (upstreamError) {
debug(`detectMetaInfo: error fetching ${upstreamUri}: ${upstreamError.status}`);
log(`detectMetaInfo: error fetching ${upstreamUri}: ${upstreamError.status}`);
return null;
}
@@ -125,21 +125,21 @@ async function detectMetaInfo(upstreamUri) {
if (favicon) {
favicon = new URL(favicon, upstreamRespose.url).toString();
debug(`detectMetaInfo: found icon: ${favicon}`);
log(`detectMetaInfo: found icon: ${favicon}`);
const [error, response] = await safe(superagent.get(favicon).timeout(10*1000));
if (error) debug(`Failed to fetch icon ${favicon}: ${error.message} ${error.status}`);
if (error) log(`Failed to fetch icon ${favicon}: ${error.message} ${error.status}`);
else if (response.headers['content-type']?.indexOf('image') !== -1) icon = response.body || response.text;
else debug(`Failed to fetch icon ${favicon}: status=${response.status}`);
else log(`Failed to fetch icon ${favicon}: status=${response.status}`);
}
if (!favicon) {
debug(`Unable to find a suitable icon for ${upstreamUri}, try fallback favicon.ico`);
log(`Unable to find a suitable icon for ${upstreamUri}, try fallback favicon.ico`);
const [error, response] = await safe(superagent.get(`${upstreamUri}/favicon.ico`).timeout(10*1000));
if (error) debug(`Failed to fetch icon ${favicon}: ${error.message}`);
if (error) log(`Failed to fetch icon ${favicon}: ${error.message}`);
else if (response.headers['content-type']?.indexOf('image') !== -1) icon = response.body || response.text;
else debug(`Failed to fetch icon ${favicon}: status=${response.status} content type ${response.headers['content-type']}`);
else log(`Failed to fetch icon ${favicon}: status=${response.status} content type ${response.headers['content-type']}`);
}
// detect label
@@ -153,7 +153,7 @@ async function detectMetaInfo(upstreamUri) {
async function add(applink) {
assert.strictEqual(typeof applink, 'object');
debug(`add: ${applink.upstreamUri}`);
log(`add: ${applink.upstreamUri}`);
let error = validateUpstreamUri(applink.upstreamUri);
if (error) throw error;

60
src/apps.js
View File

@@ -10,7 +10,7 @@ import crypto from 'node:crypto';
import { CronTime } from 'cron';
import dashboard from './dashboard.js';
import database from './database.js';
import debugModule from 'debug';
import logger from './logger.js';
import dns from './dns.js';
import docker from './docker.js';
import domains from './domains.js';
@@ -40,7 +40,7 @@ import util from 'node:util';
import volumes from './volumes.js';
import _ from './underscore.js';
const debug = debugModule('box:apps');
const { log, trace } = logger('apps');
const shell = shellModule('apps');
const PORT_TYPE_TCP = 'tcp';
@@ -306,7 +306,7 @@ function getDuplicateErrorDetails(errorMessage, locations, portBindings) {
const match = errorMessage.match(/Duplicate entry '(.*)' for key '(.*)'/);
if (!match) {
debug('Unexpected SQL error message.', errorMessage);
log('Unexpected SQL error message.', errorMessage);
return new BoxError(BoxError.DATABASE_ERROR, errorMessage);
}
@@ -1058,7 +1058,7 @@ async function onTaskFinished(error, appId, installationState, taskId, auditSour
switch (installationState) {
case ISTATE_PENDING_DATA_DIR_MIGRATION:
if (success) await safe(services.rebuildService('sftp', auditSource), { debug });
if (success) await safe(services.rebuildService('sftp', auditSource), { debug: log });
break;
case ISTATE_PENDING_UPDATE: {
const fromManifest = success ? task.args[1].updateConfig.manifest : app.manifest;
@@ -1071,8 +1071,8 @@ async function onTaskFinished(error, appId, installationState, taskId, auditSour
}
// this can race with an new install task but very unlikely
debug(`onTaskFinished: checking to stop unused services. hasPending: ${appTaskManager.hasPendingTasks()}`)
if (!appTaskManager.hasPendingTasks()) safe(services.stopUnusedServices(), { debug });
log(`onTaskFinished: checking to stop unused services. hasPending: ${appTaskManager.hasPendingTasks()}`)
if (!appTaskManager.hasPendingTasks()) safe(services.stopUnusedServices(), { debug: log });
}
async function getCount() {
@@ -1452,7 +1452,7 @@ async function startExec(app, execId, options) {
// there is a race where resizing too early results in a 404 "no such exec"
// https://git.cloudron.io/cloudron/box/issues/549
setTimeout(async function () {
await safe(docker.resizeExec(execId, { h: options.rows, w: options.columns }, { debug }));
await safe(docker.resizeExec(execId, { h: options.rows, w: options.columns }, { debug: log }));
}, 2000);
}
@@ -1568,7 +1568,7 @@ async function uploadFile(app, sourceFilePath, destFilePath) {
// the built-in bash printf understands "%q" but not /usr/bin/printf.
// ' gets replaced with '\'' . the first closes the quote and last one starts a new one
const escapedDestFilePath = await shell.bash(`printf %q '${destFilePath.replace(/'/g, '\'\\\'\'')}'`, { encoding: 'utf8' });
debug(`uploadFile: ${sourceFilePath} -> ${escapedDestFilePath}`);
log(`uploadFile: ${sourceFilePath} -> ${escapedDestFilePath}`);
const execId = await createExec(app, { cmd: [ 'bash', '-c', `cat - > ${escapedDestFilePath}` ], tty: false });
const destStream = await startExec(app, execId, { tty: false });
@@ -1702,9 +1702,9 @@ async function scheduleTask(appId, installationState, taskId, auditSource) {
const options = { timeout: 20 * 60 * 60 * 1000 /* 20 hours */, nice: 15, memoryLimit };
appTaskManager.scheduleTask(appId, taskId, options, async function (error) {
debug(`scheduleTask: task ${taskId} of ${appId} completed. error: %o`, error);
log(`scheduleTask: task ${taskId} of ${appId} completed. error: %o`, error);
if (error?.code === tasks.ECRASHED || error?.code === tasks.ESTOPPED) { // if task crashed, update the error
debug(`Apptask crashed/stopped: ${error.message}`);
log(`Apptask crashed/stopped: ${error.message}`);
const appError = {
message: error.message,
reason: BoxError.TASK_ERROR,
@@ -1713,12 +1713,12 @@ async function scheduleTask(appId, installationState, taskId, auditSource) {
taskId,
installationState
};
await safe(update(appId, { installationState: ISTATE_ERROR, error: appError, taskId: null }), { debug });
await safe(update(appId, { installationState: ISTATE_ERROR, error: appError, taskId: null }), { debug: log });
} else if (!(installationState === ISTATE_PENDING_UNINSTALL && !error)) { // clear out taskId except for successful uninstall
await safe(update(appId, { taskId: null }), { debug });
await safe(update(appId, { taskId: null }), { debug: log });
}
await safe(onTaskFinished(error, appId, installationState, taskId, auditSource), { debug }); // ignore error
await safe(onTaskFinished(error, appId, installationState, taskId, auditSource), { debug: log }); // ignore error
});
}
@@ -1740,7 +1740,7 @@ async function addTask(appId, installationState, task, auditSource) {
if (updateError && updateError.reason === BoxError.NOT_FOUND) throw new BoxError(BoxError.BAD_STATE, 'Another task is scheduled for this app'); // could be because app went away OR a taskId exists
if (updateError) throw updateError;
if (scheduleNow) await safe(scheduleTask(appId, installationState, taskId, auditSource), { debug }); // ignore error
if (scheduleNow) await safe(scheduleTask(appId, installationState, taskId, auditSource), { debug: log }); // ignore error
return taskId;
}
@@ -1882,7 +1882,7 @@ async function install(data, auditSource) {
if (constants.DEMO && (await getCount() >= constants.DEMO_APP_LIMIT)) throw new BoxError(BoxError.BAD_STATE, 'Too many installed apps, please uninstall a few and try again');
const appId = crypto.randomUUID();
debug(`Installing app ${appId}`);
log(`Installing app ${appId}`);
const app = {
accessRestriction,
@@ -2566,7 +2566,7 @@ async function exportApp(app, backupSiteId, auditSource) {
if (!canBackupApp(app)) throw new BoxError(BoxError.BAD_STATE, 'App cannot be backed up in this state');
const taskId = await tasks.add(`${tasks.TASK_APP_BACKUP_PREFIX}${app.id}`, [ appId, backupSiteId, { snapshotOnly: true } ]);
safe(tasks.startTask(taskId, {}), { debug }); // background
safe(tasks.startTask(taskId, {}), { debug: log }); // background
return { taskId };
}
@@ -2887,7 +2887,7 @@ async function getBackupDownloadStream(app, backupId) {
const stream = await backupSites.storageApi(backupSite).download(backupSite.config, downloadBackup.remotePath);
stream.on('error', function(error) {
debug(`getBackupDownloadStream: read stream error: ${error.message}`);
log(`getBackupDownloadStream: read stream error: ${error.message}`);
ps.emit('error', new BoxError(BoxError.EXTERNAL_ERROR, error));
});
stream.pipe(ps);
@@ -2926,11 +2926,11 @@ async function restoreApps(apps, options, auditSource) {
requireNullTaskId: false // ignore existing stale taskId
};
debug(`restoreApps: marking ${app.fqdn} as ${installationState} using restore config ${JSON.stringify(restoreConfig)}`);
log(`restoreApps: marking ${app.fqdn} as ${installationState} using restore config ${JSON.stringify(restoreConfig)}`);
const [addTaskError, taskId] = await safe(addTask(app.id, installationState, task, auditSource));
if (addTaskError) debug(`restoreApps: error marking ${app.fqdn} for restore: ${JSON.stringify(addTaskError)}`);
else debug(`restoreApps: marked ${app.id} as ${installationState} with taskId ${taskId}`);
if (addTaskError) log(`restoreApps: error marking ${app.fqdn} for restore: ${JSON.stringify(addTaskError)}`);
else log(`restoreApps: marked ${app.id} as ${installationState} with taskId ${taskId}`);
}
}
@@ -2945,7 +2945,7 @@ async function configureApps(apps, options, auditSource) {
const scheduleNow = !!options.scheduleNow;
for (const app of apps) {
debug(`configureApps: marking ${app.fqdn} for reconfigure (scheduleNow: ${scheduleNow})`);
log(`configureApps: marking ${app.fqdn} for reconfigure (scheduleNow: ${scheduleNow})`);
const task = {
args: {},
@@ -2955,8 +2955,8 @@ async function configureApps(apps, options, auditSource) {
};
const [addTaskError, taskId] = await safe(addTask(app.id, ISTATE_PENDING_CONFIGURE, task, auditSource));
if (addTaskError) debug(`configureApps: error marking ${app.fqdn} for configure: ${JSON.stringify(addTaskError)}`);
else debug(`configureApps: marked ${app.id} for re-configure with taskId ${taskId}`);
if (addTaskError) log(`configureApps: error marking ${app.fqdn} for configure: ${JSON.stringify(addTaskError)}`);
else log(`configureApps: marked ${app.id} for re-configure with taskId ${taskId}`);
}
}
@@ -2972,7 +2972,7 @@ async function restartAppsUsingAddons(changedAddons, auditSource) {
apps = apps.filter(app => app.runState !== RSTATE_STOPPED); // don't start stopped apps
for (const app of apps) {
debug(`restartAppsUsingAddons: marking ${app.fqdn} for restart`);
log(`restartAppsUsingAddons: marking ${app.fqdn} for restart`);
const task = {
args: {},
@@ -2981,27 +2981,27 @@ async function restartAppsUsingAddons(changedAddons, auditSource) {
// stop apps before updating the databases because postgres will "lock" them preventing import
const [stopError] = await safe(docker.stopContainers(app.id));
if (stopError) debug(`restartAppsUsingAddons: error stopping ${app.fqdn}`, stopError);
if (stopError) log(`restartAppsUsingAddons: error stopping ${app.fqdn}`, stopError);
const [addTaskError, taskId] = await safe(addTask(app.id, ISTATE_PENDING_RESTART, task, auditSource));
if (addTaskError) debug(`restartAppsUsingAddons: error marking ${app.fqdn} for restart: ${JSON.stringify(addTaskError)}`);
else debug(`restartAppsUsingAddons: marked ${app.id} for restart with taskId ${taskId}`);
if (addTaskError) log(`restartAppsUsingAddons: error marking ${app.fqdn} for restart: ${JSON.stringify(addTaskError)}`);
else log(`restartAppsUsingAddons: marked ${app.id} for restart with taskId ${taskId}`);
}
}
async function schedulePendingTasks(auditSource) {
assert.strictEqual(typeof auditSource, 'object');
debug('schedulePendingTasks: scheduling app tasks');
log('schedulePendingTasks: scheduling app tasks');
const result = await list();
for (const app of result) {
if (!app.taskId) continue; // if not in any pending state, do nothing
debug(`schedulePendingTasks: schedule task for ${app.fqdn} ${app.id}: state=${app.installationState},taskId=${app.taskId}`);
log(`schedulePendingTasks: schedule task for ${app.fqdn} ${app.id}: state=${app.installationState},taskId=${app.taskId}`);
await safe(scheduleTask(app.id, app.installationState, app.taskId, auditSource), { debug }); // ignore error
await safe(scheduleTask(app.id, app.installationState, app.taskId, auditSource), { debug: log }); // ignore error
}
}

24
src/appstore.js
View File

@@ -4,7 +4,7 @@ import backupSites from './backupsites.js';
import BoxError from './boxerror.js';
import constants from './constants.js';
import dashboard from './dashboard.js';
import debugModule from 'debug';
import logger from './logger.js';
import domains from './domains.js';
import dockerRegistries from './dockerregistries.js';
import directoryServer from './directoryserver.js';
@@ -23,7 +23,7 @@ import system from './system.js';
import users from './users.js';
import volumes from './volumes.js';
const debug = debugModule('box:appstore');
const { log, trace } = logger('appstore');
// These are the default options and will be adjusted once a subscription state is obtained
// Keep in sync with appstore/routes/cloudrons.js
@@ -129,7 +129,7 @@ async function getSubscription() {
if (!token) throw new BoxError(BoxError.LICENSE_ERROR, 'Missing token');
const [stateError, state] = await safe(getState());
if (stateError) debug('getSubscription: error getting current state', stateError);
if (stateError) log('getSubscription: error getting current state', stateError);
const [error, response] = await safe(superagent.post(`${await getApiServerOrigin()}/api/v1/subscription3`)
.query({ accessToken: token })
@@ -157,8 +157,8 @@ async function getSubscription() {
// cron hook
async function checkSubscription() {
const [error, result] = await safe(getSubscription());
if (error) debug('checkSubscription error:', error);
else debug(`checkSubscription: Cloudron ${result.cloudronId} is on the ${result.plan.name} plan.`);
if (error) log('checkSubscription error:', error);
else log(`checkSubscription: Cloudron ${result.cloudronId} is on the ${result.plan.name} plan.`);
}
function isFreePlan(subscription) {
@@ -236,7 +236,7 @@ async function getAppUpdate(app, options) {
// do some sanity checks
if (!safe.query(updateInfo, 'manifest.version') || semver.gt(curAppVersion, safe.query(updateInfo, 'manifest.version'))) {
debug('Skipping malformed update of app %s version: %s. got %j', app.id, curAppVersion, updateInfo);
log('Skipping malformed update of app %s version: %s. got %j', app.id, curAppVersion, updateInfo);
throw new BoxError(BoxError.EXTERNAL_ERROR, `Malformed update: ${response.status} ${response.text}`);
}
@@ -268,7 +268,7 @@ async function updateCloudron(data) {
if (response.status === 401) throw new BoxError(BoxError.LICENSE_ERROR, 'Invalid appstore token');
if (response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, `Bad response: ${response.status} ${response.text}`);
debug(`updateCloudron: Cloudron updated with data ${JSON.stringify(data)}`);
log(`updateCloudron: Cloudron updated with data ${JSON.stringify(data)}`);
}
async function registerCloudron3() {
@@ -277,7 +277,7 @@ async function registerCloudron3() {
const token = await settings.get(settings.APPSTORE_API_TOKEN_KEY);
if (token) { // when installed using setupToken, this updates the domain record when called during provisioning
debug('registerCloudron3: already registered. Just updating the record.');
log('registerCloudron3: already registered. Just updating the record.');
await getSubscription();
return await updateCloudron({ domain, version });
}
@@ -296,7 +296,7 @@ async function registerCloudron3() {
await settings.set(settings.CLOUDRON_ID_KEY, response.body.cloudronId);
await settings.set(settings.APPSTORE_API_TOKEN_KEY, response.body.cloudronToken);
debug(`registerCloudron3: Cloudron registered with id ${response.body.cloudronId}`);
log(`registerCloudron3: Cloudron registered with id ${response.body.cloudronId}`);
await getSubscription();
}
@@ -307,7 +307,7 @@ async function unregister() {
}
async function unlinkAccount() {
debug('unlinkAccount: Unlinking existing account.');
log('unlinkAccount: Unlinking existing account.');
if (constants.DEMO) throw new BoxError(BoxError.BAD_STATE, 'Not allowed in demo mode');
@@ -326,7 +326,7 @@ async function downloadManifest(appStoreId, manifest) {
const url = await getApiServerOrigin() + '/api/v1/apps/' + id + (version ? '/versions/' + version : '');
debug(`downloading manifest from ${url}`);
log(`downloading manifest from ${url}`);
const [error, response] = await safe(superagent.get(url).timeout(60 * 1000).ok(() => true));
@@ -388,7 +388,7 @@ async function getApp(appId) {
async function downloadIcon(appStoreId, version) {
const iconUrl = `${await getApiServerOrigin()}/api/v1/apps/${appStoreId}/versions/${version}/icon`;
return await promiseRetry({ times: 10, interval: 5000, debug }, async function () {
return await promiseRetry({ times: 10, interval: 5000, debug: log }, async function () {
const [networkError, response] = await safe(superagent.get(iconUrl)
.timeout(60 * 1000)
.ok(() => true));

34
src/apptask.js
View File

@@ -9,7 +9,7 @@ import backuptask from './backuptask.js';
import BoxError from './boxerror.js';
import community from './community.js';
import constants from './constants.js';
import debugModule from 'debug';
import logger from './logger.js';
import df from './df.js';
import dns from './dns.js';
import docker from './docker.js';
@@ -28,7 +28,7 @@ import services from './services.js';
import shellModule from './shell.js';
import _ from './underscore.js';
const debug = debugModule('box:apptask');
const { log, trace } = logger('apptask');
const shell = shellModule('apptask');
const LOGROTATE_CONFIG_EJS = fs.readFileSync(import.meta.dirname + '/logrotate.ejs', { encoding: 'utf8' }),
@@ -63,7 +63,7 @@ async function allocateContainerIp(app) {
if (app.manifest.id === constants.PROXY_APP_APPSTORE_ID) return;
await promiseRetry({ times: 10, interval: 0, debug }, async function () {
await promiseRetry({ times: 10, interval: 0, debug: log }, async function () {
const iprange = iputils.intFromIp(constants.APPS_IPv4_END) - iputils.intFromIp(constants.APPS_IPv4_START);
const rnd = Math.floor(Math.random() * iprange);
const containerIp = iputils.ipFromInt(iputils.intFromIp(constants.APPS_IPv4_START) + rnd);
@@ -92,7 +92,7 @@ async function deleteAppDir(app, options) {
const resolvedAppDataDir = stat.isSymbolicLink() ? safe.fs.readlinkSync(appDataDir) : appDataDir;
debug(`deleteAppDir - removing files in ${resolvedAppDataDir}`);
log(`deleteAppDir - removing files in ${resolvedAppDataDir}`);
if (safe.fs.existsSync(resolvedAppDataDir)) {
const entries = safe.fs.readdirSync(resolvedAppDataDir);
@@ -105,7 +105,7 @@ async function deleteAppDir(app, options) {
const entryStat = safe.fs.statSync(fullPath);
if (entryStat && !entryStat.isDirectory()) {
safe.fs.unlinkSync(fullPath);
debug(`deleteAppDir - ${fullPath} ${safe.error?.message || ''}`);
log(`deleteAppDir - ${fullPath} ${safe.error?.message || ''}`);
}
}
}
@@ -135,7 +135,7 @@ async function deleteContainers(app, options) {
assert.strictEqual(typeof app, 'object');
assert.strictEqual(typeof options, 'object');
debug('deleteContainer: deleting app containers (app, scheduler)');
log('deleteContainer: deleting app containers (app, scheduler)');
// remove configs that rely on container id
await removeLogrotateConfig(app);
@@ -149,7 +149,7 @@ async function cleanupLogs(app) {
// note that redis container logs are cleaned up by the addon
const [error] = await safe(fs.promises.rm(path.join(paths.LOG_DIR, app.id), { force: true, recursive: true }));
if (error) debug('cleanupLogs: cannot cleanup logs: %o', error);
if (error) log('cleanupLogs: cannot cleanup logs: %o', error);
}
async function verifyManifest(manifest) {
@@ -167,10 +167,10 @@ async function downloadIcon(app) {
let packageIcon = null;
if (app.versionsUrl && app.manifest.iconUrl) {
debug(`downloadIcon: Downloading community icon ${app.manifest.iconUrl}`);
log(`downloadIcon: Downloading community icon ${app.manifest.iconUrl}`);
packageIcon = await community.downloadIcon(app.manifest);
} else if (app.appStoreId) {
debug(`downloadIcon: Downloading icon of ${app.appStoreId}@${app.manifest.version}`);
log(`downloadIcon: Downloading icon of ${app.appStoreId}@${app.manifest.version}`);
packageIcon = await appstore.downloadIcon(app.appStoreId, app.manifest.version);
}
@@ -251,7 +251,7 @@ async function updateChecklist(app, newChecks, acknowledged = false) {
}
async function startApp(app) {
debug('startApp: starting container');
log('startApp: starting container');
if (app.runState === apps.RSTATE_STOPPED) return;
@@ -378,7 +378,7 @@ async function createContainer(app) {
if (app.manifest.id === constants.PROXY_APP_APPSTORE_ID) return;
debug('createContainer: creating container');
log('createContainer: creating container');
const container = await docker.createContainer(app);
@@ -808,7 +808,7 @@ async function run(appId, args, progressCallback) {
const app = await apps.get(appId);
debug(`run: startTask installationState: ${app.installationState} runState: ${app.runState}`);
log(`run: startTask installationState: ${app.installationState} runState: ${app.runState}`);
let cmd;
@@ -855,19 +855,19 @@ async function run(appId, args, progressCallback) {
cmd = updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null });
break;
default:
debug('run: apptask launched with invalid command');
log('run: apptask launched with invalid command');
throw new BoxError(BoxError.INTERNAL_ERROR, 'Unknown install command in apptask:' + app.installationState);
}
const [error, result] = await safe(cmd); // only some commands like backup return a result
if (error) {
debug(`run: app error for state ${app.installationState}: %o`, error);
log(`run: app error for state ${app.installationState}: %o`, error);
if (app.installationState === apps.ISTATE_PENDING_UPDATE && error.backupError) {
debug('run: update aborted because backup failed');
await safe(updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null }, { debug }));
log('run: update aborted because backup failed');
await safe(updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null }, { debug: log }));
} else {
await safe(updateApp(app, { installationState: apps.ISTATE_ERROR, error: makeTaskError(error, app) }), { debug });
await safe(updateApp(app, { installationState: apps.ISTATE_ERROR, error: makeTaskError(error, app) }), { debug: log });
}
throw error;

12
src/apptaskmanager.js
View File

@@ -1,6 +1,6 @@
import assert from 'node:assert';
import BoxError from './boxerror.js';
import debugModule from 'debug';
import logger from './logger.js';
import fs from 'node:fs';
import locks from './locks.js';
import path from 'node:path';
@@ -9,7 +9,7 @@ import safe from 'safetydance';
import scheduler from './scheduler.js';
import tasks from './tasks.js';
const debug = debugModule('box:apptaskmanager');
const { log, trace } = logger('apptaskmanager');
const gActiveTasks = {}; // indexed by app id
@@ -22,12 +22,12 @@ const DRAIN_TIMER_SECS = 1000;
let gDrainTimerId = null;
async function drain() {
debug(`drain: ${gPendingTasks.length} apptasks pending`);
log(`drain: ${gPendingTasks.length} apptasks pending`);
for (let i = 0; i < gPendingTasks.length; i++) {
const space = Object.keys(gActiveTasks).length - TASK_CONCURRENCY;
if (space == 0) {
debug('At concurrency limit, cannot drain anymore');
log('At concurrency limit, cannot drain anymore');
break;
}
@@ -53,7 +53,7 @@ async function drain() {
.catch((error) => { taskError = error; })
.finally(async () => {
delete gActiveTasks[appId];
await safe(onFinished(taskError, taskResult), { debug }); // hasPendingTasks() can now return false
await safe(onFinished(taskError, taskResult), { debug: log }); // hasPendingTasks() can now return false
await locks.release(`${locks.TYPE_APP_TASK_PREFIX}${appId}`);
await locks.releaseByTaskId(taskId);
scheduler.resumeAppJobs(appId);
@@ -68,7 +68,7 @@ async function start() {
assert.strictEqual(gDrainTimerId, null);
assert.strictEqual(gStarted, false);
debug('started');
log('started');
gStarted = true;
if (gPendingTasks.length) gDrainTimerId = setTimeout(drain, DRAIN_TIMER_SECS);

10
src/asynctask.js
View File

@@ -1,8 +1,8 @@
import debugModule from 'debug';
import logger from './logger.js';
import EventEmitter from 'node:events';
import safe from 'safetydance';
const debug = debugModule('box:asynctask');
const { log, trace } = logger('asynctask');
// this runs in-process
class AsyncTask extends EventEmitter {
@@ -20,16 +20,16 @@ class AsyncTask extends EventEmitter {
}
async start() { // should not throw!
debug(`start: ${this.name} started`);
log(`start: ${this.name} started`);
const [error] = await safe(this._run(this.#abortController.signal));
debug(`start: ${this.name} finished`);
log(`start: ${this.name} finished`);
this.emit('done', { errorMessage: error?.message || '' });
this.#abortController = null;
}
stop() {
if (this.#abortController === null) return; // already finished
debug(`stop: ${this.name} . sending abort signal`);
log(`stop: ${this.name} . sending abort signal`);
this.#abortController.abort();
}

44
src/backupcleaner.js
View File

@@ -6,12 +6,12 @@ import backups from './backups.js';
import backupFormats from './backupformats.js';
import backupSites from './backupsites.js';
import constants from './constants.js';
import debugModule from 'debug';
import logger from './logger.js';
import moment from 'moment';
import path from 'node:path';
import safe from 'safetydance';
const debug = debugModule('box:backupcleaner');
const { log, trace } = logger('backupcleaner');
function applyBackupRetention(allBackups, retention, referencedBackupIds) {
assert(Array.isArray(allBackups));
@@ -67,7 +67,7 @@ function applyBackupRetention(allBackups, retention, referencedBackupIds) {
}
for (const backup of allBackups) {
debug(`applyBackupRetention: ${backup.remotePath} keep/discard: ${backup.keepReason || backup.discardReason || 'unprocessed'}`);
log(`applyBackupRetention: ${backup.remotePath} keep/discard: ${backup.keepReason || backup.discardReason || 'unprocessed'}`);
}
}
@@ -88,21 +88,21 @@ async function removeBackup(site, backup, progressCallback) {
}
if (removeError) {
debug(`removeBackup: error removing backup ${removeError.message}`);
log(`removeBackup: error removing backup ${removeError.message}`);
return;
}
// remove integrity info
const [removeIntegrityError] = await safe(backupSites.storageApi(site).remove(site.config, `${remotePath}.backupinfo`));
if (removeIntegrityError) debug(`removeBackup: could not remove integrity info: ${removeIntegrityError.message}`);
if (removeIntegrityError) log(`removeBackup: could not remove integrity info: ${removeIntegrityError.message}`);
// prune empty directory if possible
const [pruneError] = await safe(backupSites.storageApi(site).remove(site.config, path.dirname(remotePath)));
if (pruneError) debug(`removeBackup: unable to prune backup directory ${path.dirname(remotePath)}: ${pruneError.message}`);
if (pruneError) log(`removeBackup: unable to prune backup directory ${path.dirname(remotePath)}: ${pruneError.message}`);
const [delError] = await safe(backups.del(backup.id));
if (delError) debug(`removeBackup: error removing ${backup.id} from database. %o`, delError);
else debug(`removeBackup: removed ${backup.remotePath}`);
if (delError) log(`removeBackup: error removing ${backup.id} from database. %o`, delError);
else log(`removeBackup: removed ${backup.remotePath}`);
}
async function cleanupAppBackups(site, referencedBackupIds, progressCallback) {
@@ -129,7 +129,7 @@ async function cleanupAppBackups(site, referencedBackupIds, progressCallback) {
let appBackupsToRemove = [];
for (const appId of Object.keys(appBackupsById)) {
const appRetention = Object.assign({ keepLatest: allAppIds.includes(appId) }, site.retention);
debug(`cleanupAppBackups: applying retention for appId ${appId} retention: ${JSON.stringify(appRetention)}`);
log(`cleanupAppBackups: applying retention for appId ${appId} retention: ${JSON.stringify(appRetention)}`);
applyBackupRetention(appBackupsById[appId], appRetention, referencedBackupIds);
appBackupsToRemove = appBackupsToRemove.concat(appBackupsById[appId].filter(b => !b.keepReason));
}
@@ -140,7 +140,7 @@ async function cleanupAppBackups(site, referencedBackupIds, progressCallback) {
await removeBackup(site, appBackup, progressCallback); // never errors
}
debug('cleanupAppBackups: done');
log('cleanupAppBackups: done');
return removedAppBackupPaths;
}
@@ -163,7 +163,7 @@ async function cleanupMailBackups(site, referencedBackupIds, progressCallback) {
await removeBackup(site, mailBackup, progressCallback); // never errors
}
debug('cleanupMailBackups: done');
log('cleanupMailBackups: done');
return removedMailBackupPaths;
}
@@ -193,7 +193,7 @@ async function cleanupBoxBackups(site, progressCallback) {
await removeBackup(site, boxBackup, progressCallback);
}
debug('cleanupBoxBackups: done');
log('cleanupBoxBackups: done');
return { removedBoxBackupPaths, referencedBackupIds };
}
@@ -223,7 +223,7 @@ async function cleanupMissingBackups(site, progressCallback) {
await progressCallback({ message: `Removing missing backup ${backup.remotePath}`});
const [delError] = await safe(backups.del(backup.id));
if (delError) debug(`cleanupMissingBackups: error removing ${backup.id} from database. %o`, delError);
if (delError) log(`cleanupMissingBackups: error removing ${backup.id} from database. %o`, delError);
missingBackupPaths.push(backup.remotePath);
}
@@ -231,7 +231,7 @@ async function cleanupMissingBackups(site, progressCallback) {
++ page;
} while (result.length === perPage);
debug('cleanupMissingBackups: done');
log('cleanupMissingBackups: done');
return missingBackupPaths;
}
@@ -242,7 +242,7 @@ async function removeOldAppSnapshots(site) {
const snapshotInfo = await backupSites.getSnapshotInfo(site);
const progressCallback = (progress) => { debug(`removeOldAppSnapshots: ${progress.message}`); };
const progressCallback = (progress) => { log(`removeOldAppSnapshots: ${progress.message}`); };
for (const appId of Object.keys(snapshotInfo)) {
if (appId === 'box' || appId === 'mail') continue;
@@ -253,16 +253,16 @@ async function removeOldAppSnapshots(site) {
const ext = backupFormats.api(site.format).getFileExtension(!!site.encryption);
const remotePath = `snapshot/app_${appId}${ext}`;
if (ext) {
await safe(backupSites.storageApi(site).remove(site.config, remotePath), { debug });
await safe(backupSites.storageApi(site).remove(site.config, remotePath), { debug: log });
} else {
await safe(backupSites.storageApi(site).removeDir(site.config, site.limits, remotePath, progressCallback), { debug });
await safe(backupSites.storageApi(site).removeDir(site.config, site.limits, remotePath, progressCallback), { debug: log });
}
await backupSites.setSnapshotInfo(site, appId, null /* info */);
debug(`removeOldAppSnapshots: removed snapshot of app ${appId}`);
log(`removeOldAppSnapshots: removed snapshot of app ${appId}`);
}
debug('removeOldAppSnapshots: done');
log('removeOldAppSnapshots: done');
}
async function run(siteId, progressCallback) {
@@ -272,14 +272,14 @@ async function run(siteId, progressCallback) {
const site = await backupSites.get(siteId);
if (!site) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Target not found');
debug(`run: retention is ${JSON.stringify(site.retention)}`);
log(`run: retention is ${JSON.stringify(site.retention)}`);
const status = await backupSites.ensureMounted(site);
debug(`run: mount point status is ${JSON.stringify(status)}`);
log(`run: mount point status is ${JSON.stringify(status)}`);
if (status.state !== 'active') throw new BoxError(BoxError.MOUNT_ERROR, `Backup endpoint is not mounted: ${status.message}`);
if (site.retention.keepWithinSecs < 0) {
debug('run: keeping all backups');
log('run: keeping all backups');
return {};
}

40
src/backupformat/rsync.js
View File

@@ -3,7 +3,7 @@ import async from 'async';
import backupSites from '../backupsites.js';
import BoxError from '../boxerror.js';
import DataLayout from '../datalayout.js';
import debugModule from 'debug';
import logger from '../logger.js';
import hush from '../hush.js';
const { DecryptStream, EncryptStream } = hush;
import fs from 'node:fs';
@@ -19,7 +19,7 @@ import syncer from '../syncer.js';
import util from 'node:util';
import { Writable } from 'node:stream';
const debug = debugModule('box:backupformat/rsync');
const { log, trace } = logger('backupformat/rsync');
const shell = shellModule('backupformat/rsync');
async function addFile(sourceFile, encryption, uploader, progressCallback) {
@@ -32,7 +32,7 @@ async function addFile(sourceFile, encryption, uploader, progressCallback) {
// destinations dirs/file which are owned by root (this process id) and cannot be copied (run as normal user)
const [openError, sourceHandle] = await safe(fs.promises.open(sourceFile, 'r'));
if (openError) {
debug(`addFile: ignoring disappeared file: ${sourceFile}`);
log(`addFile: ignoring disappeared file: ${sourceFile}`);
return { integrity: null, stats: { transferred: 0 } };
}
@@ -61,7 +61,7 @@ async function addFile(sourceFile, encryption, uploader, progressCallback) {
const [error] = await safe(pipelinePromise);
if (error && !error.message.includes('ENOENT')) throw new BoxError(BoxError.EXTERNAL_ERROR, `tarPack pipeline error for ${sourceFile}: ${error.message}`); // ignore error if file disappears
// debug(`addFile: pipeline finished: ${JSON.stringify(ps.stats())}`);
// log(`addFile: pipeline finished: ${JSON.stringify(ps.stats())}`);
await uploader.finish();
@@ -112,7 +112,7 @@ async function restoreFsMetadata(dataLayout, metadataFile) {
assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
assert.strictEqual(typeof metadataFile, 'string');
debug(`Recreating empty directories in ${dataLayout.toString()}`);
log(`Recreating empty directories in ${dataLayout.toString()}`);
const metadataJson = safe.fs.readFileSync(metadataFile, 'utf8');
if (metadataJson === null) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Error loading fsmetadata.json:' + safe.error.message);
@@ -164,7 +164,7 @@ async function sync(backupSite, remotePath, dataLayout, progressCallback) {
const concurrency = backupSite.limits?.syncConcurrency || (backupSite.provider === 's3' ? 20 : 10);
const cacheFile = path.join(paths.BACKUP_INFO_DIR, backupSite.id, `${dataLayout.getBasename()}.sync.cache`);
const { delQueue, addQueue, integrityMap } = await syncer.sync(dataLayout, cacheFile); // integrityMap is unchanged files
debug(`sync: processing ${delQueue.length} deletes, ${addQueue.length} additions and ${integrityMap.size} unchanged`);
log(`sync: processing ${delQueue.length} deletes, ${addQueue.length} additions and ${integrityMap.size} unchanged`);
const aggregatedStats = {
transferred: 0,
size: [...integrityMap.values()].reduce((sum, integrity) => sum + (integrity?.size || 0), 0), // integrity can be null if file had disappeared during upload
@@ -193,9 +193,9 @@ async function sync(backupSite, remotePath, dataLayout, progressCallback) {
} else if (change.operation === 'remove') {
await backupSites.storageApi(backupSite).remove(backupSite.config, fullPath);
} else if (change.operation === 'add') {
await promiseRetry({ times: 5, interval: 20000, debug }, async (retryCount) => {
await promiseRetry({ times: 5, interval: 20000, debug: log }, async (retryCount) => {
reportUploadProgress(`Current: ${change.path}` + (retryCount > 1 ? ` (Try ${retryCount})` : ''));
if (retryCount > 1) debug(`sync: retrying ${change.path} position ${change.position} try ${retryCount}`);
if (retryCount > 1) log(`sync: retrying ${change.path} position ${change.position} try ${retryCount}`);
const uploader = await backupSites.storageApi(backupSite).upload(backupSite.config, backupSite.limits, fullPath);
const result = await addFile(dataLayout.toLocalPath('./' + change.path), backupSite.encryption, uploader, (progress) => {
reportUploadProgress(progress.message);
@@ -210,11 +210,11 @@ async function sync(backupSite, remotePath, dataLayout, progressCallback) {
}
const [delError] = await safe(async.eachLimit(delQueue, concurrency, async (change) => await processSyncerChange(change, backupSite, remotePath, dataLayout, progressCallback)));
debug('sync: done processing deletes. error: %o', delError);
log('sync: done processing deletes. error: %o', delError);
if (delError) throw delError;
const [addError] = await safe(async.eachLimit(addQueue, concurrency, async (change) => await processSyncerChange(change, backupSite, remotePath, dataLayout, progressCallback)));
debug('sync: done processing adds. error: %o', addError);
log('sync: done processing adds. error: %o', addError);
if (addError) throw addError;
progressCallback({ message: `Uploaded ${completedAdds} files` });
@@ -235,7 +235,7 @@ async function downloadDir(backupSite, remotePath, dataLayout, progressCallback)
const encryptedFilenames = backupSite.encryption?.encryptedFilenames || false;
debug(`downloadDir: ${remotePath} to ${dataLayout.toString()}. encryption filenames: ${encryptedFilenames}. encrypted files: ${!!backupSite.encryption}`);
log(`downloadDir: ${remotePath} to ${dataLayout.toString()}. encryption filenames: ${encryptedFilenames}. encrypted files: ${!!backupSite.encryption}`);
let completedFiles = 0, totalFiles = 0;
let lastProgressTime = 0;
@@ -264,7 +264,7 @@ async function downloadDir(backupSite, remotePath, dataLayout, progressCallback)
const [downloadError, sourceStream] = await safe(backupSites.storageApi(backupSite).download(backupSite.config, entry.path));
if (downloadError) {
debug(`downloadDir: download ${entry.path} to ${destFilePath} errored: ${downloadError.message}`);
log(`downloadDir: download ${entry.path} to ${destFilePath} errored: ${downloadError.message}`);
throw downloadError;
}
@@ -290,7 +290,7 @@ async function downloadDir(backupSite, remotePath, dataLayout, progressCallback)
const [pipelineError] = await safe(pipeline(streams));
if (pipelineError) {
debug(`downloadDir: download error ${entry.path} to ${destFilePath}: ${pipelineError.message}`);
log(`downloadDir: download error ${entry.path} to ${destFilePath}: ${pipelineError.message}`);
throw pipelineError;
}
});
@@ -318,7 +318,7 @@ async function download(backupSite, remotePath, dataLayout, progressCallback) {
assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
assert.strictEqual(typeof progressCallback, 'function');
debug(`download: Downloading ${remotePath} to ${dataLayout.toString()}`);
log(`download: Downloading ${remotePath} to ${dataLayout.toString()}`);
await downloadDir(backupSite, remotePath, dataLayout, progressCallback);
await restoreFsMetadata(dataLayout, `${dataLayout.localRoot()}/fsmetadata.json`);
@@ -330,7 +330,7 @@ async function upload(backupSite, remotePath, dataLayout, progressCallback) {
assert.strictEqual(typeof dataLayout, 'object');
assert.strictEqual(typeof progressCallback, 'function');
debug(`upload: uploading to site ${backupSite.id} path ${remotePath} (encrypted: ${!!backupSite.encryption}) dataLayout ${dataLayout.toString()}`);
log(`upload: uploading to site ${backupSite.id} path ${remotePath} (encrypted: ${!!backupSite.encryption}) dataLayout ${dataLayout.toString()}`);
await saveFsMetadata(dataLayout, `${dataLayout.localRoot()}/fsmetadata.json`);
return await sync(backupSite, remotePath, dataLayout, progressCallback); // { stats, integrityMap }
@@ -370,7 +370,7 @@ async function verify(backupSite, remotePath, integrityMap, progressCallback) {
assert(util.types.isMap(integrityMap), 'integrityMap should be a Map');
assert.strictEqual(typeof progressCallback, 'function');
debug(`verify: Verifying ${remotePath}`);
log(`verify: Verifying ${remotePath}`);
// https://www.digitalocean.com/community/questions/rate-limiting-on-spaces?answer=40441
const concurrency = backupSite.limits?.downloadConcurrency || (backupSite.provider === 's3' ? 30 : 10);
@@ -386,12 +386,12 @@ async function verify(backupSite, remotePath, integrityMap, progressCallback) {
const integrity = integrityMap.get(relativePath);
if (result.transferred !== integrity.size) {
messages.push(`${entry.path} has size ${result.transferred}. Expecting ${integrity.size}`);
debug(`verify: size check of ${entry.path} failed: ${messages.at(-1)}`);
log(`verify: size check of ${entry.path} failed: ${messages.at(-1)}`);
} else if (result.digest !== integrity.sha256) {
messages.push(`${entry.path} has digest ${result.digest}. Expecting ${integrity.sha256}`);
debug(`verify: digest check of ${entry.path} failed: ${messages.at(-1)}`);
log(`verify: digest check of ${entry.path} failed: ${messages.at(-1)}`);
} else {
debug(`verify: ${entry.path} passed`);
log(`verify: ${entry.path} passed`);
}
});
fileCount += batch.entries.length;
@@ -401,7 +401,7 @@ async function verify(backupSite, remotePath, integrityMap, progressCallback) {
if (integrityMap.size !== fileCount) {
messages.push(`Got ${fileCount} files. Expecting ${integrityMap.size} files`);
debug(`verify: file count mismatch: ${messages.at(-1)}`);
log(`verify: file count mismatch: ${messages.at(-1)}`);
}
return messages;

56
src/backupformat/tgz.js
View File

@@ -2,7 +2,7 @@ import assert from 'node:assert';
import backupSites from '../backupsites.js';
import BoxError from '../boxerror.js';
import DataLayout from '../datalayout.js';
import debugModule from 'debug';
import logger from '../logger.js';
import hush from '../hush.js';
const { DecryptStream, EncryptStream } = hush;
import fs from 'node:fs';
@@ -17,7 +17,7 @@ import tar from 'tar-stream';
import util from 'node:util';
import zlib from 'node:zlib';
const debug = debugModule('box:backupformat/tgz');
const { log, trace } = logger('backupformat/tgz');
// In tar, the entry header contains the file size. If we don't provide it those many bytes, the tar will become corrupt
// Linux provides no guarantee of how many bytes can be read from a file. This is the case with sqlite and log files
@@ -31,12 +31,12 @@ class EnsureFileSizeStream extends Transform {
_transform(chunk, encoding, callback) {
if (this._remaining <= 0) {
debug(`EnsureFileSizeStream: ${this._name} dropping ${chunk.length} bytes`);
log(`EnsureFileSizeStream: ${this._name} dropping ${chunk.length} bytes`);
return callback(null);
}
if (this._remaining - chunk.length < 0) {
debug(`EnsureFileSizeStream: ${this._name} dropping extra ${chunk.length - this._remaining} bytes`);
log(`EnsureFileSizeStream: ${this._name} dropping extra ${chunk.length - this._remaining} bytes`);
chunk = chunk.subarray(0, this._remaining);
this._remaining = 0;
} else {
@@ -48,7 +48,7 @@ class EnsureFileSizeStream extends Transform {
_flush(callback) {
if (this._remaining > 0) {
debug(`EnsureFileSizeStream: ${this._name} injecting ${this._remaining} bytes`);
log(`EnsureFileSizeStream: ${this._name} injecting ${this._remaining} bytes`);
this.push(Buffer.alloc(this._remaining, 0));
}
callback();
@@ -63,7 +63,7 @@ function addEntryToPack(pack, header, options) {
return new Promise((resolve, reject) => {
const packEntry = safe(() => pack.entry(header, function (error) {
if (error) {
debug(`addToPack: error adding ${header.name} ${header.type} ${error.message}`);
log(`addToPack: error adding ${header.name} ${header.type} ${error.message}`);
reject(new BoxError(BoxError.FS_ERROR, error.message));
} else {
resolve();
@@ -74,7 +74,7 @@ function addEntryToPack(pack, header, options) {
if (options?.input) {
const ensureFileSizeStream = new EnsureFileSizeStream({ name: header.name, size: header.size });
safe(stream.pipeline(options.input, ensureFileSizeStream, packEntry), { debug }); // background. rely on pack.entry callback for promise completion
safe(stream.pipeline(options.input, ensureFileSizeStream, packEntry), { debug: log }); // background. rely on pack.entry callback for promise completion
}
});
}
@@ -92,7 +92,7 @@ async function addPathToPack(pack, localPath, dataLayout) {
const dir = queue.shift();
const [readdirError, entries] = await safe(fs.promises.readdir(dir, { withFileTypes: true }));
if (!entries) {
debug(`tarPack: skipping directory ${dir}: ${readdirError.message}`);
log(`tarPack: skipping directory ${dir}: ${readdirError.message}`);
continue;
}
const subdirs = [];
@@ -101,9 +101,9 @@ async function addPathToPack(pack, localPath, dataLayout) {
const headerName = dataLayout.toRemotePath(abspath);
if (entry.isFile()) {
const [openError, handle] = await safe(fs.promises.open(abspath, 'r'));
if (!handle) { debug(`tarPack: skipping file, could not open ${abspath}: ${openError.message}`); continue; }
if (!handle) { log(`tarPack: skipping file, could not open ${abspath}: ${openError.message}`); continue; }
const [statError, stat] = await safe(handle.stat());
if (!stat) { debug(`tarPack: skipping file, could not stat ${abspath}: ${statError.message}`); continue; }
if (!stat) { log(`tarPack: skipping file, could not stat ${abspath}: ${statError.message}`); continue; }
const header = { name: headerName, type: 'file', mode: stat.mode, size: stat.size, uid: process.getuid(), gid: process.getgid() };
if (stat.size > 8589934590 || entry.name.length > 99) header.pax = { size: stat.size };
const input = handle.createReadStream({ autoClose: true });
@@ -116,12 +116,12 @@ async function addPathToPack(pack, localPath, dataLayout) {
++stats.dirCount;
} else if (entry.isSymbolicLink()) {
const [readlinkError, site] = await safe(fs.promises.readlink(abspath));
if (!site) { debug(`tarPack: skipping link, could not readlink ${abspath}: ${readlinkError.message}`); continue; }
if (!site) { log(`tarPack: skipping link, could not readlink ${abspath}: ${readlinkError.message}`); continue; }
const header = { name: headerName, type: 'symlink', linkname: site, uid: process.getuid(), gid: process.getgid() };
await addEntryToPack(pack, header, { /* options */ });
++stats.linkCount;
} else {
debug(`tarPack: ignoring unknown type ${entry.name} ${entry.type}`);
log(`tarPack: ignoring unknown type ${entry.name} ${entry.type}`);
}
}
@@ -163,11 +163,11 @@ async function tarPack(dataLayout, encryption, uploader, progressCallback) {
let fileCount = 0;
for (const localPath of dataLayout.localPaths()) {
const [error, stats] = await safe(addPathToPack(pack, localPath, dataLayout), { debug });
const [error, stats] = await safe(addPathToPack(pack, localPath, dataLayout), { debug: log });
if (error) break; // the pipeline will error and we will retry the whole packing all over
fileCount += stats.fileCount;
}
debug(`tarPack: packed ${fileCount} files`);
log(`tarPack: packed ${fileCount} files`);
pack.finalize(); // harmless to call if already in error state
@@ -175,7 +175,7 @@ async function tarPack(dataLayout, encryption, uploader, progressCallback) {
if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, `tarPack pipeline error: ${error.message}`);
const stats = ps.stats(); // { startTime, totalMsecs, transferred }
debug(`tarPack: pipeline finished: ${JSON.stringify(stats)}`);
log(`tarPack: pipeline finished: ${JSON.stringify(stats)}`);
await uploader.finish();
return {
@@ -195,7 +195,7 @@ async function tarExtract(inStream, dataLayout, encryption, progressCallback) {
let entryCount = 0;
extract.on('entry', async function (header, entryStream, next) {
if (path.isAbsolute(header.name)) {
debug(`tarExtract: ignoring absolute path ${header.name}`);
log(`tarExtract: ignoring absolute path ${header.name}`);
return next();
}
++entryCount;
@@ -211,7 +211,7 @@ async function tarExtract(inStream, dataLayout, encryption, progressCallback) {
await safe(fs.promises.unlink(abspath)); // remove any link created from previous failed extract
[error] = await safe(fs.promises.symlink(header.linkname, abspath));
} else {
debug(`tarExtract: ignoring unknown entry: ${header.name} ${header.type}`);
log(`tarExtract: ignoring unknown entry: ${header.name} ${header.type}`);
entryStream.resume(); // drain
}
@@ -220,7 +220,7 @@ async function tarExtract(inStream, dataLayout, encryption, progressCallback) {
[error] = await safe(fs.promises.lutimes(abspath, now /* atime */, header.mtime)); // for dirs, mtime will get overwritten
next(error);
});
extract.on('finish', () => debug(`tarExtract: extracted ${entryCount} entries`));
extract.on('finish', () => log(`tarExtract: extracted ${entryCount} entries`));
const gunzip = zlib.createGunzip({});
const ps = new ProgressStream({ interval: 10000 });
@@ -242,7 +242,7 @@ async function tarExtract(inStream, dataLayout, encryption, progressCallback) {
if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, `tarExtract pipeline error: ${error.message}`);
}
debug(`tarExtract: pipeline finished: ${JSON.stringify(ps.stats())}`);
log(`tarExtract: pipeline finished: ${JSON.stringify(ps.stats())}`);
}
async function download(backupSite, remotePath, dataLayout, progressCallback) {
@@ -251,9 +251,9 @@ async function download(backupSite, remotePath, dataLayout, progressCallback) {
assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
assert.strictEqual(typeof progressCallback, 'function');
debug(`download: Downloading ${remotePath} to ${dataLayout.toString()}`);
log(`download: Downloading ${remotePath} to ${dataLayout.toString()}`);
await promiseRetry({ times: 3, interval: 20000, debug }, async () => {
await promiseRetry({ times: 3, interval: 20000, debug: log }, async () => {
progressCallback({ message: `Downloading backup ${remotePath}` });
const sourceStream = await backupSites.storageApi(backupSite).download(backupSite.config, remotePath);
@@ -267,9 +267,9 @@ async function upload(backupSite, remotePath, dataLayout, progressCallback) {
assert.strictEqual(typeof dataLayout, 'object');
assert.strictEqual(typeof progressCallback, 'function');
debug(`upload: uploading to site ${backupSite.id} path ${remotePath} (encrypted: ${!!backupSite.encryption}) dataLayout ${dataLayout.toString()}`);
log(`upload: uploading to site ${backupSite.id} path ${remotePath} (encrypted: ${!!backupSite.encryption}) dataLayout ${dataLayout.toString()}`);
return await promiseRetry({ times: 5, interval: 20000, debug }, async () => {
return await promiseRetry({ times: 5, interval: 20000, debug: log }, async () => {
progressCallback({ message: `Uploading backup ${remotePath}` });
const uploader = await backupSites.storageApi(backupSite).upload(backupSite.config, backupSite.limits, remotePath);
@@ -296,7 +296,7 @@ async function verify(backupSite, remotePath, integrityMap, progressCallback) {
assert(util.types.isMap(integrityMap), 'integrityMap should be a Map');
assert.strictEqual(typeof progressCallback, 'function');
debug(`verify: Verifying ${remotePath}`);
log(`verify: Verifying ${remotePath}`);
const inStream = await backupSites.storageApi(backupSite).download(backupSite.config, remotePath);
@@ -305,17 +305,17 @@ async function verify(backupSite, remotePath, integrityMap, progressCallback) {
const extract = tar.extract();
extract.on('entry', async function (header, entryStream, next) {
if (path.isAbsolute(header.name)) {
debug(`verify: ignoring absolute path ${header.name}`);
log(`verify: ignoring absolute path ${header.name}`);
return next();
}
debug(`verify: ${header.name} ${header.size} ${header.type}`);
log(`verify: ${header.name} ${header.size} ${header.type}`);
if (header.type === 'file') {
++fileCount;
}
entryStream.resume(); // drain
next();
});
extract.on('finish', () => debug('verify: extract finished'));
extract.on('finish', () => log('verify: extract finished'));
const hash = new HashStream();
const gunzip = zlib.createGunzip({});
@@ -336,7 +336,7 @@ async function verify(backupSite, remotePath, integrityMap, progressCallback) {
}
const integrity = integrityMap.get('.');
debug(`verify: Expecting: ${JSON.stringify(integrity)} Actual: size:${ps.stats().transferred} filecount:${fileCount} digest:${hash.digest()}`);
log(`verify: Expecting: ${JSON.stringify(integrity)} Actual: size:${ps.stats().transferred} filecount:${fileCount} digest:${hash.digest()}`);
const messages = [];
if (integrity.size !== ps.stats().transferred) messages.push(`Size mismatch. Expected: ${integrity.size} Actual: ${ps.stats().transferred}`);

6
src/backupintegrity.js
View File

@@ -5,10 +5,10 @@ import backupSites from './backupsites.js';
import BoxError from './boxerror.js';
import consumers from 'node:stream/consumers';
import crypto from 'node:crypto';
import debugModule from 'debug';
import logger from './logger.js';
import safe from 'safetydance';
const debug = debugModule('box:backupintegrity');
const { log, trace } = logger('backupintegrity');
async function downloadBackupInfo(backupSite, backup) {
@@ -47,7 +47,7 @@ async function verify(backup, backupSite, progressCallback) {
if (verifyError) messages.push(`Failed to verify ${backup.remotePath}: ${verifyError.message}`);
if (verifyMessages) messages.push(...verifyMessages);
debug(`verified: ${backup.remotePath} ${JSON.stringify(messages, null, 4)}`);
log(`verified: ${backup.remotePath} ${JSON.stringify(messages, null, 4)}`);
stats.duration = Date.now() - stats.startTime;
return { stats, messages: messages.slice(0, 50) }; // keep rsync fails to 50 to not overflow db

8
src/backups.js
View File

@@ -1,13 +1,13 @@
import assert from 'node:assert';
import BoxError from './boxerror.js';
import database from './database.js';
import debugModule from 'debug';
import logger from './logger.js';
import eventlog from './eventlog.js';
import hat from './hat.js';
import safe from 'safetydance';
import tasks from './tasks.js';
const debug = debugModule('box:backups');
const { log, trace } = logger('backups');
const BACKUP_TYPE_APP = 'app';
const BACKUP_STATE_NORMAL = 'normal';
@@ -222,11 +222,11 @@ async function startIntegrityCheck(backup, auditSource) {
// background
tasks.startTask(taskId, {})
.then(async (status) => {
debug(`startIntegrityCheck: task completed`);
log(`startIntegrityCheck: task completed`);
await eventlog.add(eventlog.ACTION_BACKUP_INTEGRITY_FINISH, auditSource, { status, taskId, backupId: backup.id });
})
.catch(async (error) => {
debug(`startIntegrityCheck: task error. ${error.message}`);
log(`startIntegrityCheck: task error. ${error.message}`);
await eventlog.add(eventlog.ACTION_BACKUP_INTEGRITY_FINISH, auditSource, { errorMessage: error.message, taskId, backupId: backup.id });
});

20
src/backupsites.js
View File

@@ -6,7 +6,7 @@ import cron from './cron.js';
import { CronTime } from 'cron';
import crypto from 'node:crypto';
import database from './database.js';
import debugModule from 'debug';
import logger from './logger.js';
import eventlog from './eventlog.js';
import hush from './hush.js';
import locks from './locks.js';
@@ -18,7 +18,7 @@ import storageFilesystem from './storage/filesystem.js';
import storageS3 from './storage/s3.js';
import storageGcs from './storage/gcs.js';
const debug = debugModule('box:backups');
const { log, trace } = logger('backups');
// format: rsync or tgz
@@ -304,7 +304,7 @@ async function del(backupSite, auditSource) {
assert.strictEqual(typeof backupSite, 'object');
assert.strictEqual(typeof auditSource, 'object');
await safe(storageApi(backupSite).teardown(backupSite.config), { debug }); // ignore error
await safe(storageApi(backupSite).teardown(backupSite.config), { debug: log }); // ignore error
const queries = [
{ query: 'DELETE FROM archives WHERE backupId IN (SELECT id FROM backups WHERE siteId=?)', args: [ backupSite.id ] },
@@ -437,12 +437,12 @@ async function setConfig(backupSite, newConfig, auditSource) {
newConfig = structuredClone(newConfig); // make a copy
storageApi(backupSite).injectPrivateFields(newConfig, oldConfig);
debug('setConfig: validating new storage configuration');
log('setConfig: validating new storage configuration');
const sanitizedConfig = await storageApi(backupSite).verifyConfig({ id: backupSite.id, provider: backupSite.provider, config: newConfig });
await update(backupSite, { config: sanitizedConfig });
debug('setConfig: setting up new storage configuration');
log('setConfig: setting up new storage configuration');
await storageApi(backupSite).setup(sanitizedConfig);
await eventlog.add(eventlog.ACTION_BACKUP_SITE_UPDATE, auditSource, { name: backupSite.name, config: storageApi(backupSite).removePrivateFields(newConfig) });
@@ -487,13 +487,13 @@ async function add(data, auditSource) {
const id = crypto.randomUUID();
if (!safe.fs.mkdirSync(`${paths.BACKUP_INFO_DIR}/${id}`)) throw new BoxError(BoxError.FS_ERROR, `Failed to create info dir: ${safe.error.message}`);
debug('add: validating new storage configuration');
log('add: validating new storage configuration');
const sanitizedConfig = await storageApi({ provider }).verifyConfig({id, provider, config });
await database.query('INSERT INTO backupSites (id, name, provider, configJson, contentsJson, limitsJson, integrityKeyPairJson, retentionJson, schedule, encryptionJson, format, enableForUpdates) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
[ id, name, provider, JSON.stringify(sanitizedConfig), JSON.stringify(contents), JSON.stringify(limits), JSON.stringify(integrityKeyPair), JSON.stringify(retention), schedule, JSON.stringify(encryption), format, enableForUpdates ]);
debug('add: setting up new storage configuration');
log('add: setting up new storage configuration');
await storageApi({ provider }).setup(sanitizedConfig);
await eventlog.add(eventlog.ACTION_BACKUP_SITE_ADD, auditSource, { id, name, provider, contents, schedule, format });
@@ -504,7 +504,7 @@ async function add(data, auditSource) {
async function addDefault(auditSource) {
assert.strictEqual(typeof auditSource, 'object');
debug('addDefault: adding default backup site');
log('addDefault: adding default backup site');
const defaultBackupSite = {
name: 'Default',
provider: 'filesystem',
@@ -536,7 +536,7 @@ async function createPseudo(data) {
encryption.encryptionPasswordHint = '';
}
debug('add: validating new storage configuration');
log('add: validating new storage configuration');
const sanitizedConfig = await storageApi({ provider }).verifyConfig({id, provider, config });
return { id, format, provider, config: sanitizedConfig, encryption };
}
@@ -547,7 +547,7 @@ async function reinitAll() {
if (!safe.fs.mkdirSync(`${paths.BACKUP_INFO_DIR}/${site.id}`, { recursive: true })) throw new BoxError(BoxError.FS_ERROR, `Failed to create info dir: ${safe.error.message}`);
const status = await getStatus(site);
if (status.state === 'active') continue;
safe(remount(site), { debug }); // background
safe(remount(site), { debug: log }); // background
}
}

64
src/backuptask.js
View File

@@ -8,7 +8,7 @@ import constants from './constants.js';
import crypto from 'node:crypto';
import DataLayout from './datalayout.js';
import database from './database.js';
import debugModule from 'debug';
import logger from './logger.js';
import df from './df.js';
import locks from './locks.js';
import path from 'node:path';
@@ -20,7 +20,7 @@ import shellModule from './shell.js';
import stream from 'stream/promises';
import util from 'util';
const debug = debugModule('box:backuptask');
const { log, trace } = logger('backuptask');
const shell = shellModule('backuptask');
@@ -40,22 +40,22 @@ async function checkPreconditions(backupSite, dataLayout) {
// check mount status before uploading
const status = await backupSites.ensureMounted(backupSite);
debug(`checkPreconditions: mount point status is ${JSON.stringify(status)}`);
log(`checkPreconditions: mount point status is ${JSON.stringify(status)}`);
if (status.state !== 'active') throw new BoxError(BoxError.MOUNT_ERROR, `Backup endpoint is not active: ${status.message}`);
// check availabe size. this requires root for df to work
const available = await backupSites.storageApi(backupSite).getAvailableSize(backupSite.config);
let used = 0;
for (const localPath of dataLayout.localPaths()) {
debug(`checkPreconditions: getting disk usage of ${localPath}`);
log(`checkPreconditions: getting disk usage of ${localPath}`);
// du can error when files go missing as it is computing the size. it still prints some size anyway
// to match df output in getAvailableSize() we must use disk usage size here and not apparent size
const [duError, result] = await safe(shell.spawn('du', [ '--dereference-args', '--summarize', '--block-size=1', '--exclude=*.lock', '--exclude=dovecot.list.index.log.*', localPath], { encoding: 'utf8' }));
if (duError) debug(`checkPreconditions: du error for ${localPath}. code: ${duError.code} stderror: ${duError.stderr}`);
if (duError) log(`checkPreconditions: du error for ${localPath}. code: ${duError.code} stderror: ${duError.stderr}`);
used += parseInt(duError ? duError.stdout : result, 10);
}
debug(`checkPreconditions: total required=${used} available=${available}`);
log(`checkPreconditions: total required=${used} available=${available}`);
const needed = 0.6 * used + (1024 * 1024 * 1024); // check if there is atleast 1GB left afterwards. aim for 60% because rsync/tgz won't need full 100%
if (available <= needed) throw new BoxError(BoxError.FS_ERROR, `Not enough disk space for backup. Needed: ${df.prettyBytes(needed)} Available: ${df.prettyBytes(available)}`);
@@ -81,7 +81,7 @@ async function upload(remotePath, siteId, dataLayoutString, progressCallback) {
assert.strictEqual(typeof dataLayoutString, 'string');
assert.strictEqual(typeof progressCallback, 'function');
debug(`upload: path ${remotePath} site ${siteId} dataLayout ${dataLayoutString}`);
log(`upload: path ${remotePath} site ${siteId} dataLayout ${dataLayoutString}`);
const backupSite = await backupSites.get(siteId);
if (!backupSite) throw new BoxError(BoxError.NOT_FOUND, 'Backup site not found');
@@ -99,7 +99,7 @@ async function upload(remotePath, siteId, dataLayoutString, progressCallback) {
// - rsync: size (final backup size) will be different from what was transferred (only changed files)
// stats.fileCount and stats.size are stored in db and should match up what is written into .backupinfo
const { stats, integrityMap } = await backupFormats.api(backupSite.format).upload(backupSite, remotePath, dataLayout, progressCallback);
debug(`upload: path ${remotePath} site ${siteId} uploaded: ${JSON.stringify(stats)}`);
log(`upload: path ${remotePath} site ${siteId} uploaded: ${JSON.stringify(stats)}`);
progressCallback({ message: `Uploading integrity information to ${remotePath}.backupinfo` });
const signature = await uploadBackupInfo(backupSite, remotePath, integrityMap);
@@ -112,7 +112,7 @@ async function download(backupSite, remotePath, dataLayout, progressCallback) {
assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
assert.strictEqual(typeof progressCallback, 'function');
debug(`download: Downloading ${remotePath} of format ${backupSite.format} (encrypted: ${!!backupSite.encryption}) to ${dataLayout.toString()}`);
log(`download: Downloading ${remotePath} of format ${backupSite.format} (encrypted: ${!!backupSite.encryption}) to ${dataLayout.toString()}`);
await backupFormats.api(backupSite.format).download(backupSite, remotePath, dataLayout, progressCallback);
}
@@ -128,10 +128,10 @@ async function restore(backupSite, remotePath, progressCallback) {
await download(backupSite, remotePath, dataLayout, progressCallback);
debug('restore: download completed, importing database');
log('restore: download completed, importing database');
await database.importFromFile(`${dataLayout.localRoot()}/box.mysqldump`);
debug('restore: database imported');
log('restore: database imported');
await locks.releaseAll(); // clear the locks table in database
}
@@ -155,7 +155,7 @@ async function downloadApp(app, restoreConfig, progressCallback) {
}
await download(backupSite, remotePath, dataLayout, progressCallback);
debug('downloadApp: time: %s', (new Date() - startTime)/1000);
log('downloadApp: time: %s', (new Date() - startTime)/1000);
}
async function runBackupUpload(uploadConfig, progressCallback) {
@@ -172,21 +172,21 @@ async function runBackupUpload(uploadConfig, progressCallback) {
const envCopy = Object.assign({}, process.env);
if (backupSite.limits?.memoryLimit >= 2*1024*1024*1024) {
const heapSize = Math.min((backupSite.limits.memoryLimit/1024/1024) - 256, 8192);
debug(`runBackupUpload: adjusting heap size to ${heapSize}M`);
log(`runBackupUpload: adjusting heap size to ${heapSize}M`);
envCopy.NODE_OPTIONS = `--max-old-space-size=${heapSize}`;
}
let lastMessage = null; // the script communicates error result as a string
function onMessage(progress) { // this is { message } or { result }
if ('message' in progress) return progressCallback({ message: `${progress.message} (${progressTag})` });
debug(`runBackupUpload: result - ${JSON.stringify(progress)}`);
log(`runBackupUpload: result - ${JSON.stringify(progress)}`);
lastMessage = progress;
}
// do not use debug for logging child output because it already has timestamps via it's own debug
const [error] = await safe(shell.sudo([ BACKUP_UPLOAD_CMD, remotePath, backupSite.id, dataLayout.toString() ], { env: envCopy, preserveEnv: true, onMessage, logger: process.stdout.write }));
if (error && (error.code === null /* signal */ || (error.code !== 0 && error.code !== 50))) { // backuptask crashed
debug(`runBackupUpload: backuptask crashed`, error);
log(`runBackupUpload: backuptask crashed`, error);
throw new BoxError(BoxError.INTERNAL_ERROR, 'Backuptask crashed');
} else if (error && error.code === 50) { // exited with error
throw new BoxError(BoxError.EXTERNAL_ERROR, lastMessage.errorMessage);
@@ -203,7 +203,7 @@ async function snapshotBox(progressCallback) {
const startTime = new Date();
await database.exportToFile(`${paths.BOX_DATA_DIR}/box.mysqldump`);
debug(`snapshotBox: took ${(new Date() - startTime)/1000} seconds`);
log(`snapshotBox: took ${(new Date() - startTime)/1000} seconds`);
}
async function uploadBoxSnapshot(backupSite, progressCallback) {
@@ -230,7 +230,7 @@ async function uploadBoxSnapshot(backupSite, progressCallback) {
const { stats, integrity } = await runBackupUpload(uploadConfig, progressCallback);
debug(`uploadBoxSnapshot: took ${(new Date() - startTime)/1000} seconds`);
log(`uploadBoxSnapshot: took ${(new Date() - startTime)/1000} seconds`);
await backupSites.setSnapshotInfo(backupSite, 'box', { timestamp: new Date().toISOString() });
@@ -246,17 +246,17 @@ async function copy(backupSite, srcRemotePath, destRemotePath, progressCallback)
const startTime = new Date();
const [copyError] = await safe(backupFormats.api(backupSite.format).copy(backupSite, srcRemotePath, destRemotePath, progressCallback));
if (copyError) {
debug(`copy: copy to ${destRemotePath} errored. error: ${copyError.message}`);
log(`copy: copy to ${destRemotePath} errored. error: ${copyError.message}`);
throw copyError;
}
debug(`copy: copied successfully to ${destRemotePath}. Took ${(new Date() - startTime)/1000} seconds`);
log(`copy: copied successfully to ${destRemotePath}. Took ${(new Date() - startTime)/1000} seconds`);
const [copyChecksumError] = await safe(backupSites.storageApi(backupSite).copy(backupSite.config, `${srcRemotePath}.backupinfo`, `${destRemotePath}.backupinfo`, progressCallback));
if (copyChecksumError) {
debug(`copy: copy to ${destRemotePath} errored. error: ${copyChecksumError.message}`);
log(`copy: copy to ${destRemotePath} errored. error: ${copyChecksumError.message}`);
throw copyChecksumError;
}
debug(`copy: copied backupinfo successfully to ${destRemotePath}.backupinfo`);
log(`copy: copied backupinfo successfully to ${destRemotePath}.backupinfo`);
}
async function backupBox(backupSite, appBackupsMap, tag, options, progressCallback) {
@@ -281,7 +281,7 @@ async function backupBox(backupSite, appBackupsMap, tag, options, progressCallba
duration: acc.duration + cur.upload.duration,
}), stats.upload);
debug(`backupBox: rotating box snapshot of ${backupSite.id} to id ${remotePath}. ${JSON.stringify(stats)}`);
log(`backupBox: rotating box snapshot of ${backupSite.id} to id ${remotePath}. ${JSON.stringify(stats)}`);
const data = {
remotePath,
@@ -329,7 +329,7 @@ async function snapshotApp(app, progressCallback) {
await services.runBackupCommand(app);
await services.backupAddons(app, app.manifest.addons);
debug(`snapshotApp: ${app.fqdn} took ${(new Date() - startTime)/1000} seconds`);
log(`snapshotApp: ${app.fqdn} took ${(new Date() - startTime)/1000} seconds`);
}
async function uploadAppSnapshot(backupSite, app, progressCallback) {
@@ -358,7 +358,7 @@ async function uploadAppSnapshot(backupSite, app, progressCallback) {
const { stats, integrity } = await runBackupUpload(uploadConfig, progressCallback);
debug(`uploadAppSnapshot: ${app.fqdn} uploaded to ${remotePath}. ${(new Date() - startTime)/1000} seconds`);
log(`uploadAppSnapshot: ${app.fqdn} uploaded to ${remotePath}. ${(new Date() - startTime)/1000} seconds`);
await backupSites.setSnapshotInfo(backupSite, app.id, { timestamp: new Date().toISOString(), manifest: app.manifest });
@@ -385,7 +385,7 @@ async function backupAppWithTag(app, backupSite, tag, options, progressCallback)
const manifest = app.manifest;
const remotePath = addFileExtension(backupSite, `${tag}/app_${app.fqdn}_v${manifest.version}`);
debug(`backupAppWithTag: rotating ${app.fqdn} snapshot of ${backupSite.id} to path ${remotePath}`);
log(`backupAppWithTag: rotating ${app.fqdn} snapshot of ${backupSite.id} to path ${remotePath}`);
const data = {
remotePath,
@@ -456,7 +456,7 @@ async function uploadMailSnapshot(backupSite, progressCallback) {
const { stats, integrity } = await runBackupUpload(uploadConfig, progressCallback);
debug(`uploadMailSnapshot: took ${(new Date() - startTime)/1000} seconds`);
log(`uploadMailSnapshot: took ${(new Date() - startTime)/1000} seconds`);
await backupSites.setSnapshotInfo(backupSite, 'mail', { timestamp: new Date().toISOString() });
@@ -469,7 +469,7 @@ async function backupMailWithTag(backupSite, tag, options, progressCallback) {
assert.strictEqual(typeof options, 'object');
assert.strictEqual(typeof progressCallback, 'function');
debug(`backupMailWithTag: backing up mail with tag ${tag}`);
log(`backupMailWithTag: backing up mail with tag ${tag}`);
const uploadStartTime = Date.now();
const uploadResult = await uploadMailSnapshot(backupSite, progressCallback); // { stats, integrity }
@@ -477,7 +477,7 @@ async function backupMailWithTag(backupSite, tag, options, progressCallback) {
const remotePath = addFileExtension(backupSite, `${tag}/mail_v${constants.VERSION}`);
debug(`backupMailWithTag: rotating mail snapshot of ${backupSite.id} to ${remotePath}`);
log(`backupMailWithTag: rotating mail snapshot of ${backupSite.id} to ${remotePath}`);
const data = {
remotePath,
@@ -519,7 +519,7 @@ async function downloadMail(backupSite, remotePath, progressCallback) {
const startTime = new Date();
await download(backupSite, remotePath, dataLayout, progressCallback);
debug('downloadMail: time: %s', (new Date() - startTime)/1000);
log('downloadMail: time: %s', (new Date() - startTime)/1000);
}
// this function is called from external process. calling process is expected to have a lock
@@ -544,11 +544,11 @@ async function fullBackup(backupSiteId, options, progressCallback) {
percent += step;
if (!app.enableBackup) {
debug(`fullBackup: skipped backup ${app.fqdn} (${i+1}/${allApps.length}) since automatic backup disabled`);
log(`fullBackup: skipped backup ${app.fqdn} (${i+1}/${allApps.length}) since automatic backup disabled`);
continue; // nothing to backup
}
if (!backupSites.hasContent(backupSite, app.id)) {
debug(`fullBackup: skipped backup ${app.fqdn} (${i+1}/${allApps.length}) as it is not in site contents`);
log(`fullBackup: skipped backup ${app.fqdn} (${i+1}/${allApps.length}) as it is not in site contents`);
continue;
}
@@ -556,7 +556,7 @@ async function fullBackup(backupSiteId, options, progressCallback) {
await locks.wait(`${locks.TYPE_APP_BACKUP_PREFIX}${app.id}`);
const startTime = new Date();
const [appBackupError, appBackupResult] = await safe(backupAppWithTag(app, backupSite, tag, options, (progress) => progressCallback({ percent, message: progress.message })));
debug(`fullBackup: app ${app.fqdn} backup finished. Took ${(new Date() - startTime)/1000} seconds`);
log(`fullBackup: app ${app.fqdn} backup finished. Took ${(new Date() - startTime)/1000} seconds`);
await locks.release(`${locks.TYPE_APP_BACKUP_PREFIX}${app.id}`);
if (appBackupError) throw appBackupError;
if (appBackupResult) appBackupsMap.set(appBackupResult.id, appBackupResult.stats); // backupId can be null if in BAD_STATE and never backed up

6
src/branding.js
View File

@@ -2,13 +2,13 @@ import apps from './apps.js';
import assert from 'node:assert';
import BoxError from './boxerror.js';
import constants from './constants.js';
import debugModule from 'debug';
import logger from './logger.js';
import eventlog from './eventlog.js';
import paths from './paths.js';
import safe from 'safetydance';
import settings from './settings.js';
const debug = debugModule('box:branding');
const { log, trace } = logger('branding');
async function getCloudronName() {
@@ -28,7 +28,7 @@ async function setCloudronName(name, auditSource) {
// mark apps using oidc addon to be reconfigured
const [, installedApps] = await safe(apps.list());
await safe(apps.configureApps(installedApps.filter((a) => !!a.manifest.addons?.oidc), { scheduleNow: true }, auditSource), { debug });
await safe(apps.configureApps(installedApps.filter((a) => !!a.manifest.addons?.oidc), { scheduleNow: true }, auditSource), { debug: log });
await settings.set(settings.CLOUDRON_NAME_KEY, name);
await eventlog.add(eventlog.ACTION_BRANDING_NAME, auditSource, { name });

8
src/community.js
View File

@@ -1,12 +1,12 @@
import assert from 'node:assert';
import BoxError from './boxerror.js';
import debugModule from 'debug';
import logger from './logger.js';
import manifestFormat from '@cloudron/manifest-format';
import promiseRetry from './promise-retry.js';
import safe from 'safetydance';
import superagent from '@cloudron/superagent';
const debug = debugModule('box:community');
const { log, trace } = logger('community');
const CLOUDRON_VERSIONS_FILE = 'CloudronVersions.json';
@@ -125,7 +125,7 @@ async function downloadManifest(versionsUrl) {
if (!url.startsWith('https://')) throw new BoxError(BoxError.BAD_FIELD, 'versionsUrl must use https');
if (!version) throw new BoxError(BoxError.BAD_FIELD, 'version is required in versionsUrl (format: url@version)');
debug(`downloading manifest from ${url} version ${version}`);
log(`downloading manifest from ${url} version ${version}`);
const versionsRoot = await fetchVersionsRoot(url);
@@ -170,7 +170,7 @@ async function getAppUpdate(app, options) {
}
async function downloadIcon(manifest) {
return await promiseRetry({ times: 10, interval: 5000, debug }, async function () {
return await promiseRetry({ times: 10, interval: 5000, debug: log }, async function () {
const [networkError, response] = await safe(superagent.get(manifest.iconUrl)
.timeout(60 * 1000)
.ok(() => true));

54
src/cron.js
View File

@@ -6,7 +6,7 @@ import backupSites from './backupsites.js';
import cloudron from './cloudron.js';
import constants from './constants.js';
import { CronJob } from 'cron';
import debugModule from 'debug';
import logger from './logger.js';
import domains from './domains.js';
import dyndns from './dyndns.js';
import externalLdap from './externalldap.js';
@@ -24,7 +24,7 @@ import system from './system.js';
import updater from './updater.js';
import util from 'node:util';
const debug = debugModule('box:cron');
const { log, trace } = logger('cron');
// IMPORTANT: These patterns are together because they spin tasks which acquire a lock
// If the patterns overlap all the time, then the task may not ever get a chance to run!
@@ -78,7 +78,7 @@ function getCronSeed() {
hour = Math.floor(24 * Math.random());
minute = Math.floor(60 * Math.random());
debug(`getCronSeed: writing new cron seed file with ${hour}:${minute} to ${paths.CRON_SEED_FILE}`);
log(`getCronSeed: writing new cron seed file with ${hour}:${minute} to ${paths.CRON_SEED_FILE}`);
safe.fs.writeFileSync(paths.CRON_SEED_FILE, `${hour}:${minute}`);
}
@@ -91,7 +91,7 @@ async function handleBackupScheduleChanged(site) {
const tz = await cloudron.getTimeZone();
debug(`handleBackupScheduleChanged: schedule ${site.schedule} (${tz})`);
log(`handleBackupScheduleChanged: schedule ${site.schedule} (${tz})`);
if (gJobs.backups.has(site.id)) gJobs.backups.get(site.id).stop();
gJobs.backups.delete(site.id);
@@ -103,7 +103,7 @@ async function handleBackupScheduleChanged(site) {
onTick: async () => {
const t = await backupSites.get(site.id);
if (!t) return;
await safe(backupSites.startBackupTask(t, AuditSource.CRON), { debug });
await safe(backupSites.startBackupTask(t, AuditSource.CRON), { debug: log });
},
start: true,
timeZone: tz
@@ -116,7 +116,7 @@ async function handleAutoupdatePatternChanged(pattern) {
const tz = await cloudron.getTimeZone();
debug(`autoupdatePatternChanged: pattern - ${pattern} (${tz})`);
log(`autoupdatePatternChanged: pattern - ${pattern} (${tz})`);
if (gJobs.autoUpdater) gJobs.autoUpdater.stop();
gJobs.autoUpdater = null;
@@ -125,7 +125,7 @@ async function handleAutoupdatePatternChanged(pattern) {
gJobs.autoUpdater = CronJob.from({
cronTime: pattern,
onTick: async () => await safe(updater.autoUpdate(AuditSource.CRON), { debug }),
onTick: async () => await safe(updater.autoUpdate(AuditSource.CRON), { debug: log }),
start: true,
timeZone: tz
});
@@ -134,7 +134,7 @@ async function handleAutoupdatePatternChanged(pattern) {
function handleDynamicDnsChanged(enabled) {
assert.strictEqual(typeof enabled, 'boolean');
debug('Dynamic DNS setting changed to %s', enabled);
log('Dynamic DNS setting changed to %s', enabled);
if (gJobs.dynamicDns) gJobs.dynamicDns.stop();
gJobs.dynamicDns = null;
@@ -144,7 +144,7 @@ function handleDynamicDnsChanged(enabled) {
gJobs.dynamicDns = CronJob.from({
// until we can be smarter about actual IP changes, lets ensure it every 10minutes
cronTime: '00 */10 * * * *',
onTick: async () => { await safe(dyndns.refreshDns(AuditSource.CRON), { debug }); },
onTick: async () => { await safe(dyndns.refreshDns(AuditSource.CRON), { debug: log }); },
start: true
});
}
@@ -159,7 +159,7 @@ async function handleExternalLdapChanged(config) {
gJobs.externalLdapSyncer = CronJob.from({
cronTime: '00 00 */4 * * *', // every 4 hours
onTick: async () => await safe(externalLdap.startSyncer(AuditSource.CRON), { debug }),
onTick: async () => await safe(externalLdap.startSyncer(AuditSource.CRON), { debug: log }),
start: true
});
}
@@ -167,42 +167,42 @@ async function handleExternalLdapChanged(config) {
async function startJobs() {
const { hour, minute } = getCronSeed();
debug(`startJobs: starting cron jobs with hour ${hour} and minute ${minute}`);
log(`startJobs: starting cron jobs with hour ${hour} and minute ${minute}`);
gJobs.systemChecks = CronJob.from({
cronTime: `00 ${minute} 2 * * *`, // once a day. if you change this interval, change the notification messages with correct duration
onTick: async () => await safe(system.runSystemChecks(), { debug }),
onTick: async () => await safe(system.runSystemChecks(), { debug: log }),
start: true
});
gJobs.mailStatusCheck = CronJob.from({
cronTime: `00 ${minute} 2 * * *`, // once a day. if you change this interval, change the notification messages with correct duration
onTick: async () => await safe(mail.checkStatus(), { debug }),
onTick: async () => await safe(mail.checkStatus(), { debug: log }),
start: true
});
gJobs.diskSpaceChecker = CronJob.from({
cronTime: '00 30 * * * *', // every 30 minutes. if you change this interval, change the notification messages with correct duration
onTick: async () => await safe(system.checkDiskSpace(), { debug }),
onTick: async () => await safe(system.checkDiskSpace(), { debug: log }),
start: true
});
// this is run separately from the update itself so that the user can disable automatic updates but can still get a notification
gJobs.updateCheckerJob = CronJob.from({
cronTime: `00 ${minute} 1,5,9,13,17,21,23 * * *`,
onTick: async () => await safe(updater.checkForUpdates({ stableOnly: true }), { debug }),
onTick: async () => await safe(updater.checkForUpdates({ stableOnly: true }), { debug: log }),
start: true
});
gJobs.cleanupTokens = CronJob.from({
cronTime: '00 */30 * * * *', // every 30 minutes
onTick: async () => await safe(janitor.cleanupTokens(), { debug }),
onTick: async () => await safe(janitor.cleanupTokens(), { debug: log }),
start: true
});
gJobs.cleanupOidc = CronJob.from({
cronTime: '00 10 * * * *', // every hour ten minutes past
onTick: async () => await safe(oidcServer.cleanupExpired(), { debug }),
onTick: async () => await safe(oidcServer.cleanupExpired(), { debug: log }),
start: true
});
@@ -210,7 +210,7 @@ async function startJobs() {
cronTime: DEFAULT_CLEANUP_BACKUPS_PATTERN,
onTick: async () => {
for (const backupSite of await backupSites.list()) {
await safe(backupSites.startCleanupTask(backupSite, AuditSource.CRON), { debug });
await safe(backupSites.startCleanupTask(backupSite, AuditSource.CRON), { debug: log });
}
},
start: true
@@ -218,50 +218,50 @@ async function startJobs() {
gJobs.cleanupEventlog = CronJob.from({
cronTime: '00 */30 * * * *', // every 30 minutes
onTick: async () => await safe(eventlog.cleanup({ creationTime: new Date(Date.now() - 90 * 60 * 24 * 60 * 1000) }), { debug }), // 90 days ago
onTick: async () => await safe(eventlog.cleanup({ creationTime: new Date(Date.now() - 90 * 60 * 24 * 60 * 1000) }), { debug: log }), // 90 days ago
start: true
});
gJobs.dockerVolumeCleaner = CronJob.from({
cronTime: '00 00 */12 * * *', // every 12 hours
onTick: async () => await safe(janitor.cleanupDockerVolumes(), { debug }),
onTick: async () => await safe(janitor.cleanupDockerVolumes(), { debug: log }),
start: true
});
gJobs.schedulerSync = CronJob.from({
cronTime: constants.TEST ? '*/10 * * * * *' : '00 */1 * * * *', // every minute
onTick: async () => await safe(scheduler.sync(), { debug }),
onTick: async () => await safe(scheduler.sync(), { debug: log }),
start: true
});
// randomized per Cloudron based on hourlySeed
gJobs.certificateRenew = CronJob.from({
cronTime: `00 10 ${hour} * * *`,
onTick: async () => await safe(reverseProxy.startRenewCerts({}, AuditSource.CRON), { debug }),
onTick: async () => await safe(reverseProxy.startRenewCerts({}, AuditSource.CRON), { debug: log }),
start: true
});
gJobs.checkDomainConfigs = CronJob.from({
cronTime: `00 ${minute} 5 * * *`, // once a day
onTick: async () => await safe(domains.checkConfigs(AuditSource.CRON), { debug }),
onTick: async () => await safe(domains.checkConfigs(AuditSource.CRON), { debug: log }),
start: true
});
gJobs.appHealthMonitor = CronJob.from({
cronTime: '*/10 * * * * *', // every 10 seconds
onTick: async () => await safe(appHealthMonitor.run(10), { debug }), // 10 is the max run time
onTick: async () => await safe(appHealthMonitor.run(10), { debug: log }), // 10 is the max run time
start: true
});
gJobs.collectStats = CronJob.from({
cronTime: '*/20 * * * * *', // every 20 seconds. if you change this, change carbon config
onTick: async () => await safe(metrics.sendToGraphite(), { debug }),
onTick: async () => await safe(metrics.sendToGraphite(), { debug: log }),
start: true
});
gJobs.subscriptionChecker = CronJob.from({
cronTime: `00 ${minute} ${hour} * * *`, // once a day based on seed to randomize
onTick: async () => await safe(appstore.checkSubscription(), { debug }),
onTick: async () => await safe(appstore.checkSubscription(), { debug: log }),
start: true
});
@@ -289,7 +289,7 @@ async function stopJobs() {
async function handleTimeZoneChanged(tz) {
assert.strictEqual(typeof tz, 'string');
debug('handleTimeZoneChanged: recreating all jobs');
log('handleTimeZoneChanged: recreating all jobs');
await stopJobs();
await scheduler.deleteJobs(); // have to re-create with new tz
await startJobs();

18
src/dashboard.js
View File

@@ -4,7 +4,7 @@ import assert from 'node:assert';
import BoxError from './boxerror.js';
import branding from './branding.js';
import constants from './constants.js';
import debugModule from 'debug';
import logger from './logger.js';
import dns from './dns.js';
import externalLdap from './externalldap.js';
import eventlog from './eventlog.js';
@@ -18,7 +18,7 @@ import system from './system.js';
import tasks from './tasks.js';
import userDirectory from './user-directory.js';
const debug = debugModule('box:dashboard');
const { log, trace } = logger('dashboard');
async function getLocation() {
const domain = await settings.get(settings.DASHBOARD_DOMAIN_KEY);
@@ -33,7 +33,7 @@ async function setLocation(subdomain, domain) {
await settings.set(settings.DASHBOARD_SUBDOMAIN_KEY, subdomain);
await settings.set(settings.DASHBOARD_DOMAIN_KEY, domain);
debug(`setLocation: ${domain || '<cleared>'}`);
log(`setLocation: ${domain || '<cleared>'}`);
}
async function clearLocation() {
@@ -91,7 +91,7 @@ async function startPrepareLocation(domain, auditSource) {
assert.strictEqual(typeof domain, 'string');
assert.strictEqual(typeof auditSource, 'object');
debug(`prepareLocation: ${domain}`);
log(`prepareLocation: ${domain}`);
if (constants.DEMO) throw new BoxError(BoxError.BAD_STATE, 'Not allowed in demo mode');
@@ -105,7 +105,7 @@ async function startPrepareLocation(domain, auditSource) {
}
const taskId = await tasks.add(tasks.TASK_PREPARE_DASHBOARD_LOCATION, [ constants.DASHBOARD_SUBDOMAIN, domain, auditSource ]);
safe(tasks.startTask(taskId, {}), { debug }); // background
safe(tasks.startTask(taskId, {}), { debug: log }); // background
return taskId;
}
@@ -115,7 +115,7 @@ async function setupLocation(subdomain, domain, auditSource) {
assert.strictEqual(typeof domain, 'string');
assert.strictEqual(typeof auditSource, 'object');
debug(`setupLocation: ${domain}`);
log(`setupLocation: ${domain}`);
if (constants.DEMO) throw new BoxError(BoxError.BAD_STATE, 'Not allowed in demo mode');
@@ -131,12 +131,12 @@ async function changeLocation(subdomain, domain, auditSource) {
const oldLocation = await getLocation();
await setupLocation(subdomain, domain, auditSource);
debug(`setupLocation: notifying appstore and platform of domain change to ${domain}`);
log(`setupLocation: notifying appstore and platform of domain change to ${domain}`);
await eventlog.add(eventlog.ACTION_DASHBOARD_DOMAIN_UPDATE, auditSource, { subdomain, domain });
await safe(appstore.updateCloudron({ domain }), { debug });
await safe(appstore.updateCloudron({ domain }), { debug: log });
await platform.onDashboardLocationChanged(auditSource);
await safe(reverseProxy.removeDashboardConfig(oldLocation.subdomain, oldLocation.domain), { debug });
await safe(reverseProxy.removeDashboardConfig(oldLocation.subdomain, oldLocation.domain), { debug: log });
}
const _setLocation = setLocation;

14
src/database.js
View File

@@ -1,13 +1,13 @@
import assert from 'node:assert';
import BoxError from './boxerror.js';
import constants from './constants.js';
import debugModule from 'debug';
import logger from './logger.js';
import { execSync } from 'node:child_process';
import mysql from 'mysql2/promise';
import safe from 'safetydance';
import shellModule from './shell.js';
const debug = debugModule('box:database');
const { log, trace } = logger('database');
const shell = shellModule('database');
let gConnectionPool = null;
@@ -23,9 +23,9 @@ const gDatabase = {
async function uninitialize() {
if (!gConnectionPool) return;
await safe(gConnectionPool.end(), { debug });
await safe(gConnectionPool.end(), { debug: log });
gConnectionPool = null;
debug('pool closed');
log('pool closed');
}
async function query(...args) {
@@ -53,7 +53,7 @@ async function transaction(queries) {
connection.release(); // no await!
return results;
} catch (txError) {
await safe(connection.rollback(), { debug });
await safe(connection.rollback(), { debug: log });
connection.release(); // no await!
throw new BoxError(BoxError.DATABASE_ERROR, txError, { sqlCode: txError.code, sqlMessage: txError.sqlMessage || null });
}
@@ -95,7 +95,7 @@ async function initialize() {
// a crypto.randomUUID is 36 in length. so the value below provides for roughly 10k users
await conn.query('SET SESSION group_concat_max_len = 360000');
} catch (error) {
debug(`failed to init new db connection ${connection.threadId}:`, error); // only log. we will let the app handle the exception when it calls query()/transaction()
log(`failed to init new db connection ${connection.threadId}:`, error); // only log. we will let the app handle the exception when it calls query()/transaction()
}
});
}
@@ -129,7 +129,7 @@ async function runInTransaction(callback) {
connection.release(); // no await!
return result;
} catch (txError) {
await safe(connection.rollback(), { debug });
await safe(connection.rollback(), { debug: log });
connection.release(); // no await!
throw new BoxError(BoxError.DATABASE_ERROR, txError, { sqlCode: txError.code, sqlMessage: txError.sqlMessage || null });
}

8
src/df.js
View File

@@ -1,10 +1,10 @@
import assert from 'node:assert';
import BoxError from './boxerror.js';
import debugModule from 'debug';
import logger from './logger.js';
import safe from 'safetydance';
import shellModule from './shell.js';
const debug = debugModule('box:df');
const { log, trace } = logger('df');
const shell = shellModule('df');
@@ -35,7 +35,7 @@ function parseLine(line) {
async function filesystems() {
const [error, output] = await safe(shell.spawn('df', ['-B1', '--output=source,fstype,size,used,avail,pcent,target'], { encoding: 'utf8', timeout: 5000 }));
if (error) {
debug(`filesystems: df command failed. error: ${error}\n stdout: ${error.stdout}\n stderr: ${error.stderr}`);
log(`filesystems: df command failed. error: ${error}\n stdout: ${error.stdout}\n stderr: ${error.stderr}`);
throw new BoxError(BoxError.FS_ERROR, `Error running df: ${error.message}`);
}
@@ -54,7 +54,7 @@ async function file(filename) {
const [error, output] = await safe(shell.spawn('df', ['-B1', '--output=source,fstype,size,used,avail,pcent,target', filename], { encoding: 'utf8', timeout: 5000 }));
if (error) {
debug(`file: df command failed. error: ${error}\n stdout: ${error.stdout}\n stderr: ${error.stderr}`);
log(`file: df command failed. error: ${error}\n stdout: ${error.stdout}\n stderr: ${error.stderr}`);
throw new BoxError(BoxError.FS_ERROR, `Error running df: ${error.message}`);
}

34
src/directoryserver.js
View File

@@ -2,7 +2,7 @@ import assert from 'node:assert';
import AuditSource from './auditsource.js';
import BoxError from './boxerror.js';
import constants from './constants.js';
import debugModule from 'debug';
import logger from './logger.js';
import eventlog from './eventlog.js';
import ipaddr from './ipaddr.js';
import groups from './groups.js';
@@ -16,7 +16,7 @@ import shellModule from './shell.js';
import users from './users.js';
import util from 'node:util';
const debug = debugModule('box:directoryserver');
const { log, trace } = logger('directoryserver');
const shell = shellModule('directoryserver');
@@ -57,7 +57,7 @@ async function validateConfig(config) {
}
async function authorize(req, res, next) {
debug('authorize: ', req.connection.ldap.bindDN.toString());
log('authorize: ', req.connection.ldap.bindDN.toString());
// this is for connection attempts without previous bind
if (req.connection.ldap.bindDN.equals('cn=anonymous')) return next(new ldap.InsufficientAccessRightsError());
@@ -69,7 +69,7 @@ async function authorize(req, res, next) {
}
async function maybeRootDSE(req, res, next) {
debug(`maybeRootDSE: requested with scope:${req.scope} dn:${req.dn.toString()}`);
log(`maybeRootDSE: requested with scope:${req.scope} dn:${req.dn.toString()}`);
if (req.scope !== 'base') return next(new ldap.NoSuchObjectError()); // per the spec, rootDSE search require base scope
if (!req.dn || req.dn.toString() !== '') return next(new ldap.NoSuchObjectError());
@@ -126,7 +126,7 @@ async function userAuth(req, res, next) {
async function stop() {
if (!gServer) return;
debug('stopping server');
log('stopping server');
await util.promisify(gServer.close.bind(gServer))();
gServer = null;
@@ -192,7 +192,7 @@ function finalSend(results, req, res, next) {
// Will attach req.user if successful
async function userSearch(req, res, next) {
debug('user search: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id);
log('user search: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id);
const [error, allUsers] = await safe(users.list());
if (error) return next(new ldap.OperationsError(error.message));
@@ -248,7 +248,7 @@ async function userSearch(req, res, next) {
}
async function groupSearch(req, res, next) {
debug('group search: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id);
log('group search: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id);
const [error, allUsers] = await safe(users.list());
if (error) return next(new ldap.OperationsError(error.message));
@@ -292,10 +292,10 @@ async function start() {
const logger = {
trace: NOOP,
debug: NOOP,
info: debug,
warn: debug,
error: debug,
fatal: debug
info: log,
warn: log,
error: log,
fatal: log
};
gCertificate = await reverseProxy.getDirectoryServerCertificate();
@@ -307,11 +307,11 @@ async function start() {
});
gServer.on('error', function (error) {
debug('server startup error: %o', error);
log('server startup error: %o', error);
});
gServer.bind('ou=system,dc=cloudron', async function(req, res, next) {
debug('system bind: %s (from %s)', req.dn.toString(), req.connection.ldap.id);
log('system bind: %s (from %s)', req.dn.toString(), req.connection.ldap.id);
const config = await getConfig();
@@ -340,11 +340,11 @@ async function start() {
// just log that an attempt was made to unknown route, this helps a lot during app packaging
gServer.use(function(req, res, next) {
debug('not handled: dn %s, scope %s, filter %s (from %s)', req.dn ? req.dn.toString() : '-', req.scope, req.filter ? req.filter.toString() : '-', req.connection.ldap.id);
log('not handled: dn %s, scope %s, filter %s (from %s)', req.dn ? req.dn.toString() : '-', req.scope, req.filter ? req.filter.toString() : '-', req.connection.ldap.id);
return next();
});
debug(`starting server on port ${constants.USER_DIRECTORY_LDAPS_PORT}`);
log(`starting server on port ${constants.USER_DIRECTORY_LDAPS_PORT}`);
await util.promisify(gServer.listen.bind(gServer))(constants.USER_DIRECTORY_LDAPS_PORT, '::');
}
@@ -397,11 +397,11 @@ async function checkCertificate() {
const certificate = await reverseProxy.getDirectoryServerCertificate();
if (certificate.cert === gCertificate.cert) {
debug('checkCertificate: certificate has not changed');
log('checkCertificate: certificate has not changed');
return;
}
debug('checkCertificate: certificate changed. restarting');
log('checkCertificate: certificate changed. restarting');
await stop();
await start();
}

28
src/dns.js
View File

@@ -3,7 +3,7 @@ import assert from 'node:assert';
import BoxError from './boxerror.js';
import constants from './constants.js';
import dashboard from './dashboard.js';
import debugModule from 'debug';
import logger from './logger.js';
import domains from './domains.js';
import ipaddr from './ipaddr.js';
import mail from './mail.js';
@@ -36,7 +36,7 @@ import dnsOvh from './dns/ovh.js';
import dnsPorkbun from './dns/porkbun.js';
import dnsWildcard from './dns/wildcard.js';
const debug = debugModule('box:dns');
const { log, trace } = logger('dns');
const DNS_PROVIDERS = {
@@ -140,7 +140,7 @@ async function upsertDnsRecords(subdomain, domain, type, values) {
assert.strictEqual(typeof type, 'string');
assert(Array.isArray(values));
debug(`upsertDnsRecords: subdomain:${subdomain} domain:${domain} type:${type} values:${JSON.stringify(values)}`);
log(`upsertDnsRecords: subdomain:${subdomain} domain:${domain} type:${type} values:${JSON.stringify(values)}`);
const domainObject = await domains.get(domain);
await api(domainObject.provider).upsert(domainObject, subdomain, type, values);
@@ -152,7 +152,7 @@ async function removeDnsRecords(subdomain, domain, type, values) {
assert.strictEqual(typeof type, 'string');
assert(Array.isArray(values));
debug(`removeDnsRecords: subdomain:${subdomain} domain:${domain} type:${type} values:${JSON.stringify(values)}`);
log(`removeDnsRecords: subdomain:${subdomain} domain:${domain} type:${type} values:${JSON.stringify(values)}`);
const domainObject = await domains.get(domain);
const [error] = await safe(api(domainObject.provider).del(domainObject, subdomain, type, values));
@@ -220,7 +220,7 @@ async function registerLocation(location, options, recordType, recordValue) {
const [getError, values] = await safe(getDnsRecords(location.subdomain, location.domain, recordType));
if (getError) {
const retryable = getError.reason !== BoxError.ACCESS_DENIED && getError.reason !== BoxError.NOT_FOUND; // NOT_FOUND is when zone is not found
debug(`registerLocation: Get error. retryable: ${retryable}. ${getError.message}`);
log(`registerLocation: Get error. retryable: ${retryable}. ${getError.message}`);
throw new BoxError(getError.reason, `${location.domain}: ${getError.message}`, { domain: location, retryable });
}
@@ -232,7 +232,7 @@ async function registerLocation(location, options, recordType, recordValue) {
const [upsertError] = await safe(upsertDnsRecords(location.subdomain, location.domain, recordType, [ recordValue ]));
if (upsertError) {
const retryable = upsertError.reason === BoxError.BUSY || upsertError.reason === BoxError.EXTERNAL_ERROR;
debug(`registerLocation: Upsert error. retryable: ${retryable}. ${upsertError.message}`);
log(`registerLocation: Upsert error. retryable: ${retryable}. ${upsertError.message}`);
throw new BoxError(BoxError.EXTERNAL_ERROR, `${location.domain}: ${getError.message}`, { domain: location, retryable });
}
}
@@ -242,7 +242,7 @@ async function registerLocations(locations, options, progressCallback) {
assert.strictEqual(typeof options, 'object');
assert.strictEqual(typeof progressCallback, 'function');
debug(`registerLocations: Will register ${JSON.stringify(locations)} with options ${JSON.stringify(options)}`);
log(`registerLocations: Will register ${JSON.stringify(locations)} with options ${JSON.stringify(options)}`);
const ipv4 = await network.getIPv4();
const ipv6 = await network.getIPv6();
@@ -250,12 +250,12 @@ async function registerLocations(locations, options, progressCallback) {
for (const location of locations) {
progressCallback({ message: `Registering location ${fqdn(location.subdomain, location.domain)}` });
await promiseRetry({ times: 200, interval: 5000, debug, retry: (error) => error.retryable }, async function () {
await promiseRetry({ times: 200, interval: 5000, debug: log, retry: (error) => error.retryable }, async function () {
// cname records cannot co-exist with other records
const [getError, values] = await safe(getDnsRecords(location.subdomain, location.domain, 'CNAME'));
if (!getError && values.length === 1) {
if (!options.overwriteDns) throw new BoxError(BoxError.ALREADY_EXISTS, 'DNS CNAME record already exists', { domain: location, retryable: false });
debug(`registerLocations: removing CNAME record of ${fqdn(location.subdomain, location.domain)}`);
log(`registerLocations: removing CNAME record of ${fqdn(location.subdomain, location.domain)}`);
await removeDnsRecords(location.subdomain, location.domain, 'CNAME', values);
}
@@ -263,14 +263,14 @@ async function registerLocations(locations, options, progressCallback) {
await registerLocation(location, options, 'A', ipv4);
} else {
const [aError, aValues] = await safe(getDnsRecords(location.subdomain, location.domain, 'A'));
if (!aError && aValues.length) await safe(removeDnsRecords(location.subdomain, location.domain, 'A', aValues), { debug });
if (!aError && aValues.length) await safe(removeDnsRecords(location.subdomain, location.domain, 'A', aValues), { debug: log });
}
if (ipv6) {
await registerLocation(location, options, 'AAAA', ipv6);
} else {
const [aaaaError, aaaaValues] = await safe(getDnsRecords(location.subdomain, location.domain, 'AAAA'));
if (!aaaaError && aaaaValues.length) await safe(removeDnsRecords(location.subdomain, location.domain, 'AAAA', aaaaValues), { debug });
if (!aaaaError && aaaaValues.length) await safe(removeDnsRecords(location.subdomain, location.domain, 'AAAA', aaaaValues), { debug: log });
}
});
}
@@ -285,7 +285,7 @@ async function unregisterLocation(location, recordType, recordValue) {
if (!error || error.reason === BoxError.NOT_FOUND) return;
const retryable = error.reason === BoxError.BUSY || error.reason === BoxError.EXTERNAL_ERROR;
debug(`unregisterLocation: Error unregistering location ${recordType}. retryable: ${retryable}. ${error.message}`);
log(`unregisterLocation: Error unregistering location ${recordType}. retryable: ${retryable}. ${error.message}`);
throw new BoxError(BoxError.EXTERNAL_ERROR, `${location.domain}: ${error.message}`, { domain: location, retryable });
}
@@ -300,7 +300,7 @@ async function unregisterLocations(locations, progressCallback) {
for (const location of locations) {
progressCallback({ message: `Unregistering location: ${location.subdomain ? (location.subdomain + '.') : ''}${location.domain}` });
await promiseRetry({ times: 30, interval: 5000, debug, retry: (error) => error.retryable }, async function () {
await promiseRetry({ times: 30, interval: 5000, debug: log, retry: (error) => error.retryable }, async function () {
if (ipv4) await unregisterLocation(location, 'A', ipv4);
if (ipv6) await unregisterLocation(location, 'AAAA', ipv6);
});
@@ -364,7 +364,7 @@ async function startSyncDnsRecords(options) {
assert.strictEqual(typeof options, 'object');
const taskId = await tasks.add(tasks.TASK_SYNC_DNS_RECORDS, [ options ]);
safe(tasks.startTask(taskId, {}), { debug }); // background
safe(tasks.startTask(taskId, {}), { debug: log }); // background
return taskId;
}

20
src/dns/bunny.js
View File

@@ -1,14 +1,14 @@
import assert from 'node:assert';
import BoxError from '../boxerror.js';
import constants from '../constants.js';
import debugModule from 'debug';
import logger from '../logger.js';
import dig from '../dig.js';
import dns from '../dns.js';
import safe from 'safetydance';
import superagent from '@cloudron/superagent';
import waitForDns from './waitfordns.js';
const debug = debugModule('box:dns/bunny');
const { log, trace } = logger('dns/bunny');
const BUNNY_API = 'https://api.bunny.net';
@@ -61,7 +61,7 @@ async function getDnsRecords(domainConfig, zoneName, name, type) {
assert.strictEqual(typeof name, 'string');
assert.strictEqual(typeof type, 'string');
debug(`get: ${name} in zone ${zoneName} of type ${type}`);
log(`get: ${name} in zone ${zoneName} of type ${type}`);
const zoneId = await getZoneId(domainConfig, zoneName);
const [error, response] = await safe(superagent.get(`${BUNNY_API}/dnszone/${zoneId}`)
@@ -87,7 +87,7 @@ async function upsert(domainObject, location, type, values) {
zoneName = domainObject.zoneName,
name = dns.getName(domainObject, location, type) || '';
debug(`upsert: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
log(`upsert: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
const zoneId = await getZoneId(domainConfig, zoneName);
const records = await getDnsRecords(domainConfig, zoneName, name, type);
@@ -148,10 +148,10 @@ async function upsert(domainObject, location, type, values) {
.timeout(30 * 1000)
.ok(() => true));
if (error) debug(`upsert: error removing record ${records[j].id}: ${error.message}`);
if (error) log(`upsert: error removing record ${records[j].id}: ${error.message}`);
}
debug('upsert: completed with recordIds:%j', recordIds);
log('upsert: completed with recordIds:%j', recordIds);
}
async function get(domainObject, location, type) {
@@ -177,7 +177,7 @@ async function del(domainObject, location, type, values) {
zoneName = domainObject.zoneName,
name = dns.getName(domainObject, location, type) || '';
debug(`del: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
log(`del: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
const zoneId = await getZoneId(domainConfig, zoneName);
const records = await getDnsRecords(domainConfig, zoneName, name, type);
@@ -232,17 +232,17 @@ async function verifyDomainConfig(domainObject) {
if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers');
if (!nameservers.every(function (n) { return n.toLowerCase().indexOf('.bunny.net') !== -1; })) {
debug('verifyDomainConfig: %j does not contain Bunny NS', nameservers);
log('verifyDomainConfig: %j does not contain Bunny NS', nameservers);
if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Bunny');
}
const location = 'cloudrontestdns';
await upsert(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record added');
log('verifyDomainConfig: Test A record added');
await del(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record removed again');
log('verifyDomainConfig: Test A record removed again');
return credentials;
}

24
src/dns/cloudflare.js
View File

@@ -1,7 +1,7 @@
import assert from 'node:assert';
import BoxError from '../boxerror.js';
import constants from '../constants.js';
import debugModule from 'debug';
import logger from '../logger.js';
import dig from '../dig.js';
import dns from '../dns.js';
import safe from 'safetydance';
@@ -9,7 +9,7 @@ import superagent from '@cloudron/superagent';
import waitForDns from './waitfordns.js';
import _ from '../underscore.js';
const debug = debugModule('box:dns/cloudflare');
const { log, trace } = logger('dns/cloudflare');
// we are using latest v4 stable API https://api.cloudflare.com/#getting-started-endpoints
@@ -107,7 +107,7 @@ async function upsert(domainObject, location, type, values) {
zoneName = domainObject.zoneName,
fqdn = dns.fqdn(location, domainObject.domain);
debug('upsert: %s for zone %s of type %s with values %j', fqdn, zoneName, type, values);
log('upsert: %s for zone %s of type %s with values %j', fqdn, zoneName, type, values);
const zone = await getZoneByName(domainConfig, zoneName);
const zoneId = zone.id;
@@ -138,7 +138,7 @@ async function upsert(domainObject, location, type, values) {
data.proxied = !!domainConfig.defaultProxyStatus; // note that cloudflare will error if proxied is set for wrong record type or IP. only set at install time
}
debug(`upsert: Adding new record fqdn: ${fqdn}, zoneName: ${zoneName} proxied: ${data.proxied}`);
log(`upsert: Adding new record fqdn: ${fqdn}, zoneName: ${zoneName} proxied: ${data.proxied}`);
const [error, response] = await safe(createRequest('POST', `${CLOUDFLARE_ENDPOINT}/zones/${zoneId}/dns_records`, domainConfig)
.send(data));
@@ -147,7 +147,7 @@ async function upsert(domainObject, location, type, values) {
} else { // replace existing record
data.proxied = records[i].proxied; // preserve proxied parameter
debug(`upsert: Updating existing record fqdn: ${fqdn}, zoneName: ${zoneName} proxied: ${data.proxied}`);
log(`upsert: Updating existing record fqdn: ${fqdn}, zoneName: ${zoneName} proxied: ${data.proxied}`);
const [error, response] = await safe(createRequest('PUT', `${CLOUDFLARE_ENDPOINT}/zones/${zoneId}/dns_records/${records[i].id}`, domainConfig)
.send(data));
@@ -160,7 +160,7 @@ async function upsert(domainObject, location, type, values) {
for (let j = values.length + 1; j < records.length; j++) {
const [error] = await safe(createRequest('DELETE', `${CLOUDFLARE_ENDPOINT}/zones/${zoneId}/dns_records/${records[j].id}`, domainConfig));
if (error) debug(`upsert: error removing record ${records[j].id}: ${error.message}`);
if (error) log(`upsert: error removing record ${records[j].id}: ${error.message}`);
}
}
@@ -195,7 +195,7 @@ async function del(domainObject, location, type, values) {
if (result.length === 0) return;
const tmp = result.filter(function (record) { return values.some(function (value) { return value === record.content; }); });
debug('del: %j', tmp);
log('del: %j', tmp);
if (tmp.length === 0) return;
@@ -217,7 +217,7 @@ async function wait(domainObject, subdomain, type, value, options) {
zoneName = domainObject.zoneName,
fqdn = dns.fqdn(subdomain, domainObject.domain);
debug('wait: %s for zone %s of type %s', fqdn, zoneName, type);
log('wait: %s for zone %s of type %s', fqdn, zoneName, type);
const zone = await getZoneByName(domainConfig, zoneName);
const zoneId = zone.id;
@@ -227,7 +227,7 @@ async function wait(domainObject, subdomain, type, value, options) {
if (!dnsRecords[0].proxied) return await waitForDns(fqdn, domainObject.zoneName, type, value, options);
debug('wait: skipping wait of proxied domain');
log('wait: skipping wait of proxied domain');
// maybe we can check for dns to be cloudflare IPs? https://api.cloudflare.com/#cloudflare-ips-cloudflare-ip-details
}
@@ -268,17 +268,17 @@ async function verifyDomainConfig(domainObject) {
const zone = await getZoneByName(domainConfig, zoneName);
if (!_.isEqual(zone.name_servers.sort(), nameservers.sort())) {
debug('verifyDomainConfig: %j and %j do not match', nameservers, zone.name_servers);
log('verifyDomainConfig: %j and %j do not match', nameservers, zone.name_servers);
if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Cloudflare');
}
const location = 'cloudrontestdns';
await upsert(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record added');
log('verifyDomainConfig: Test A record added');
await del(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record removed again');
log('verifyDomainConfig: Test A record removed again');
return sanitizedConfig;
}

10
src/dns/desec.js
View File

@@ -1,7 +1,7 @@
import assert from 'node:assert';
import constants from '../constants.js';
import BoxError from '../boxerror.js';
import debugModule from 'debug';
import logger from '../logger.js';
import dig from '../dig.js';
import dns from '../dns.js';
import safe from 'safetydance';
@@ -9,7 +9,7 @@ import timers from 'timers/promises';
import superagent from '@cloudron/superagent';
import waitForDns from './waitfordns.js';
const debug = debugModule('box:dns/desec');
const { log, trace } = logger('dns/desec');
const DESEC_ENDPOINT = 'https://desec.io/api/v1';
@@ -143,17 +143,17 @@ async function verifyDomainConfig(domainObject) {
if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers');
if (!nameservers.every(function (n) { return n.toLowerCase().indexOf('.desec.') !== -1; })) {
debug('verifyDomainConfig: %j does not contains deSEC NS', nameservers);
log('verifyDomainConfig: %j does not contains deSEC NS', nameservers);
if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to deSEC');
}
const location = 'cloudrontestdns';
await upsert(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record added');
log('verifyDomainConfig: Test A record added');
await del(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record removed again');
log('verifyDomainConfig: Test A record removed again');
return credentials;
}

18
src/dns/digitalocean.js
View File

@@ -1,14 +1,14 @@
import assert from 'node:assert';
import BoxError from '../boxerror.js';
import constants from '../constants.js';
import debugModule from 'debug';
import logger from '../logger.js';
import dig from '../dig.js';
import dns from '../dns.js';
import safe from 'safetydance';
import superagent from '@cloudron/superagent';
import waitForDns from './waitfordns.js';
const debug = debugModule('box:dns/digitalocean');
const { log, trace } = logger('dns/digitalocean');
const DIGITALOCEAN_ENDPOINT = 'https://api.digitalocean.com';
@@ -34,7 +34,7 @@ async function getZoneRecords(domainConfig, zoneName, name, type) {
let nextPage = null, matchingRecords = [];
debug(`getInternal: getting dns records of ${zoneName} with ${name} and type ${type}`);
log(`getInternal: getting dns records of ${zoneName} with ${name} and type ${type}`);
do {
const url = nextPage ? nextPage : DIGITALOCEAN_ENDPOINT + '/v2/domains/' + zoneName + '/records';
@@ -70,7 +70,7 @@ async function upsert(domainObject, location, type, values) {
zoneName = domainObject.zoneName,
name = dns.getName(domainObject, location, type) || '@';
debug('upsert: %s for zone %s of type %s with values %j', name, zoneName, type, values);
log('upsert: %s for zone %s of type %s with values %j', name, zoneName, type, values);
const records = await getZoneRecords(domainConfig, zoneName, name, type);
@@ -136,10 +136,10 @@ async function upsert(domainObject, location, type, values) {
.retry(5)
.ok(() => true));
if (error) debug(`upsert: error removing record ${records[j].id}: ${error.message}`);
if (error) log(`upsert: error removing record ${records[j].id}: ${error.message}`);
}
debug('upsert: completed with recordIds:%j', recordIds);
log('upsert: completed with recordIds:%j', recordIds);
}
async function get(domainObject, location, type) {
@@ -229,17 +229,17 @@ async function verifyDomainConfig(domainObject) {
if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers');
if (nameservers.map(function (n) { return n.toLowerCase(); }).indexOf('ns1.digitalocean.com') === -1) {
debug('verifyDomainConfig: %j does not contains DO NS', nameservers);
log('verifyDomainConfig: %j does not contains DO NS', nameservers);
if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to DigitalOcean');
}
const location = 'cloudrontestdns';
await upsert(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record added');
log('verifyDomainConfig: Test A record added');
await del(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record removed again');
log('verifyDomainConfig: Test A record removed again');
return credentials;
}

20
src/dns/dnsimple.js
View File

@@ -1,14 +1,14 @@
import assert from 'node:assert';
import BoxError from '../boxerror.js';
import constants from '../constants.js';
import debugModule from 'debug';
import logger from '../logger.js';
import dig from '../dig.js';
import dns from '../dns.js';
import safe from 'safetydance';
import superagent from '@cloudron/superagent';
import waitForDns from './waitfordns.js';
const debug = debugModule('box:dns/dnsimple');
const { log, trace } = logger('dns/dnsimple');
const DNSIMPLE_API = 'https://api.dnsimple.com/v2';
@@ -70,7 +70,7 @@ async function getDnsRecords(domainConfig, zoneName, name, type) {
assert.strictEqual(typeof name, 'string');
assert.strictEqual(typeof type, 'string');
debug(`get: ${name} in zone ${zoneName} of type ${type}`);
log(`get: ${name} in zone ${zoneName} of type ${type}`);
const { accountId, zoneId } = await getZone(domainConfig, zoneName);
const [error, response] = await safe(superagent.get(`${DNSIMPLE_API}/${accountId}/zones/${zoneId}/records?name=${name}&type=${type}`)
@@ -96,7 +96,7 @@ async function upsert(domainObject, location, type, values) {
zoneName = domainObject.zoneName,
name = dns.getName(domainObject, location, type) || '';
debug(`upsert: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
log(`upsert: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
const { accountId, zoneId } = await getZone(domainConfig, zoneName);
const records = await getDnsRecords(domainConfig, zoneName, name, type);
@@ -157,10 +157,10 @@ async function upsert(domainObject, location, type, values) {
.timeout(30 * 1000)
.ok(() => true));
if (error) debug(`upsert: error removing record ${records[j].id}: ${error.message}`);
if (error) log(`upsert: error removing record ${records[j].id}: ${error.message}`);
}
debug('upsert: completed with recordIds:%j', recordIds);
log('upsert: completed with recordIds:%j', recordIds);
}
async function get(domainObject, location, type) {
@@ -186,7 +186,7 @@ async function del(domainObject, location, type, values) {
zoneName = domainObject.zoneName,
name = dns.getName(domainObject, location, type) || '';
debug(`del: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
log(`del: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
const { accountId, zoneId } = await getZone(domainConfig, zoneName);
const records = await getDnsRecords(domainConfig, zoneName, name, type);
@@ -241,17 +241,17 @@ async function verifyDomainConfig(domainObject) {
if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers');
if (!nameservers.every(function (n) { return n.toLowerCase().indexOf('dnsimple') !== -1; })) { // can be dnsimple.com or dnsimple-edge.org
debug('verifyDomainConfig: %j does not contain dnsimple NS', nameservers);
log('verifyDomainConfig: %j does not contain dnsimple NS', nameservers);
if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to dnsimple');
}
const location = 'cloudrontestdns';
await upsert(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record added');
log('verifyDomainConfig: Test A record added');
await del(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record removed again');
log('verifyDomainConfig: Test A record removed again');
return credentials;
}

16
src/dns/gandi.js
View File

@@ -1,14 +1,14 @@
import assert from 'node:assert';
import BoxError from '../boxerror.js';
import constants from '../constants.js';
import debugModule from 'debug';
import logger from '../logger.js';
import dig from '../dig.js';
import dns from '../dns.js';
import safe from 'safetydance';
import superagent from '@cloudron/superagent';
import waitForDns from './waitfordns.js';
const debug = debugModule('box:dns/gandi');
const { log, trace } = logger('dns/gandi');
const GANDI_API = 'https://dns.api.gandi.net/api/v5';
@@ -54,7 +54,7 @@ async function upsert(domainObject, location, type, values) {
zoneName = domainObject.zoneName,
name = dns.getName(domainObject, location, type) || '@';
debug(`upsert: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
log(`upsert: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
const data = {
'rrset_ttl': 300, // this is the minimum allowed
@@ -79,7 +79,7 @@ async function get(domainObject, location, type) {
zoneName = domainObject.zoneName,
name = dns.getName(domainObject, location, type) || '@';
debug(`get: ${name} in zone ${zoneName} of type ${type}`);
log(`get: ${name} in zone ${zoneName} of type ${type}`);
const [error, response] = await safe(createRequest('GET', `${GANDI_API}/domains/${zoneName}/records/${name}/${type}`, domainConfig));
@@ -101,7 +101,7 @@ async function del(domainObject, location, type, values) {
zoneName = domainObject.zoneName,
name = dns.getName(domainObject, location, type) || '@';
debug(`del: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
log(`del: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
const [error, response] = await safe(createRequest('DELETE', `${GANDI_API}/domains/${zoneName}/records/${name}/${type}`, domainConfig));
@@ -148,17 +148,17 @@ async function verifyDomainConfig(domainObject) {
if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers');
if (!nameservers.every(function (n) { return n.toLowerCase().indexOf('.gandi.net') !== -1; })) {
debug('verifyDomainConfig: %j does not contain Gandi NS', nameservers);
log('verifyDomainConfig: %j does not contain Gandi NS', nameservers);
if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Gandi');
}
const location = 'cloudrontestdns';
await upsert(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record added');
log('verifyDomainConfig: Test A record added');
await del(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record removed again');
log('verifyDomainConfig: Test A record removed again');
return credentials;
}

12
src/dns/gcdns.js
View File

@@ -1,7 +1,7 @@
import assert from 'node:assert';
import BoxError from '../boxerror.js';
import constants from '../constants.js';
import debugModule from 'debug';
import logger from '../logger.js';
import dig from '../dig.js';
import dns from '../dns.js';
import { DNS as GCDNS } from '@google-cloud/dns';
@@ -9,7 +9,7 @@ import safe from 'safetydance';
import waitForDns from './waitfordns.js';
import _ from '../underscore.js';
const debug = debugModule('box:dns/gcdns');
const { log, trace } = logger('dns/gcdns');
function removePrivateFields(domainObject) {
@@ -66,7 +66,7 @@ async function upsert(domainObject, location, type, values) {
zoneName = domainObject.zoneName,
fqdn = dns.fqdn(location, domainObject.domain);
debug('add: %s for zone %s of type %s with values %j', fqdn, zoneName, type, values);
log('add: %s for zone %s of type %s with values %j', fqdn, zoneName, type, values);
const zone = await getZoneByName(getDnsCredentials(domainConfig), zoneName);
@@ -170,16 +170,16 @@ async function verifyDomainConfig(domainObject) {
const definedNS = zone.metadata.nameServers.map(function(r) { return r.replace(/\.$/, ''); });
if (!_.isEqual(definedNS.sort(), nameservers.sort())) {
debug('verifyDomainConfig: %j and %j do not match', nameservers, definedNS);
log('verifyDomainConfig: %j and %j do not match', nameservers, definedNS);
if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Google Cloud DNS');
}
const location = 'cloudrontestdns';
await upsert(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record added');
log('verifyDomainConfig: Test A record added');
await del(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record removed again');
log('verifyDomainConfig: Test A record removed again');
return credentials;
}

16
src/dns/godaddy.js
View File

@@ -1,14 +1,14 @@
import assert from 'node:assert';
import BoxError from '../boxerror.js';
import constants from '../constants.js';
import debugModule from 'debug';
import logger from '../logger.js';
import dig from '../dig.js';
import dns from '../dns.js';
import safe from 'safetydance';
import superagent from '@cloudron/superagent';
import waitForDns from './waitfordns.js';
const debug = debugModule('box:dns/godaddy');
const { log, trace } = logger('dns/godaddy');
// const GODADDY_API_OTE = 'https://api.ote-godaddy.com/v1/domains';
@@ -37,7 +37,7 @@ async function upsert(domainObject, location, type, values) {
zoneName = domainObject.zoneName,
name = dns.getName(domainObject, location, type) || '@';
debug(`upsert: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
log(`upsert: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
const records = [];
for (const value of values) {
@@ -75,7 +75,7 @@ async function get(domainObject, location, type) {
zoneName = domainObject.zoneName,
name = dns.getName(domainObject, location, type) || '@';
debug(`get: ${name} in zone ${zoneName} of type ${type}`);
log(`get: ${name} in zone ${zoneName} of type ${type}`);
const [error, response] = await safe(superagent.get(`${GODADDY_API}/${zoneName}/records/${type}/${name}`)
.set('Authorization', `sso-key ${domainConfig.apiKey}:${domainConfig.apiSecret}`)
@@ -114,7 +114,7 @@ async function del(domainObject, location, type, values) {
zoneName = domainObject.zoneName,
name = dns.getName(domainObject, location, type) || '@';
debug(`del: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
log(`del: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
const result = await get(domainObject, location, type);
if (result.length === 0) return;
@@ -171,17 +171,17 @@ async function verifyDomainConfig(domainObject) {
if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers');
if (!nameservers.every(function (n) { return n.toLowerCase().indexOf('.domaincontrol.com') !== -1 || n.toLowerCase().indexOf('.secureserver.net') !== -1; })) {
debug('verifyDomainConfig: %j does not contain GoDaddy NS', nameservers);
log('verifyDomainConfig: %j does not contain GoDaddy NS', nameservers);
if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to GoDaddy');
}
const location = 'cloudrontestdns';
await upsert(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record added');
log('verifyDomainConfig: Test A record added');
await del(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record removed again');
log('verifyDomainConfig: Test A record removed again');
return credentials;
}

18
src/dns/hetzner.js
View File

@@ -1,14 +1,14 @@
import assert from 'node:assert';
import BoxError from '../boxerror.js';
import constants from '../constants.js';
import debugModule from 'debug';
import logger from '../logger.js';
import dig from '../dig.js';
import dns from '../dns.js';
import safe from 'safetydance';
import superagent from '@cloudron/superagent';
import waitForDns from './waitfordns.js';
const debug = debugModule('box:dns/hetzner');
const { log, trace } = logger('dns/hetzner');
const ENDPOINT = 'https://dns.hetzner.com/api/v1';
@@ -55,7 +55,7 @@ async function getZoneRecords(domainConfig, zone, name, type) {
let page = 1, matchingRecords = [];
debug(`getZoneRecords: getting dns records of ${zone.name} with ${name} and type ${type}`);
log(`getZoneRecords: getting dns records of ${zone.name} with ${name} and type ${type}`);
const perPage = 50;
@@ -94,7 +94,7 @@ async function upsert(domainObject, location, type, values) {
zoneName = domainObject.zoneName,
name = dns.getName(domainObject, location, type) || '@';
debug(`upsert: ${name} for zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
log(`upsert: ${name} for zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
const zone = await getZone(domainConfig, zoneName);
const records = await getZoneRecords(domainConfig, zone, name, type);
@@ -147,10 +147,10 @@ async function upsert(domainObject, location, type, values) {
.retry(5)
.ok(() => true));
if (error) debug(`upsert: error removing record ${records[j].id}: ${error.message}`);
if (error) log(`upsert: error removing record ${records[j].id}: ${error.message}`);
}
debug('upsert: completed');
log('upsert: completed');
}
async function get(domainObject, location, type) {
@@ -235,17 +235,17 @@ async function verifyDomainConfig(domainObject) {
// https://docs.hetzner.com/dns-console/dns/general/dns-overview#the-hetzner-online-name-servers-are
if (!nameservers.every(function (n) { return n.toLowerCase().search(/hetzner|your-server|second-ns/) !== -1; })) {
debug('verifyDomainConfig: %j does not contain Hetzner NS', nameservers);
log('verifyDomainConfig: %j does not contain Hetzner NS', nameservers);
if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Hetzner');
}
const location = 'cloudrontestdns';
await upsert(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record added');
log('verifyDomainConfig: Test A record added');
await del(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record removed again');
log('verifyDomainConfig: Test A record removed again');
return credentials;
}

16
src/dns/hetznercloud.js
View File

@@ -1,7 +1,7 @@
import assert from 'node:assert';
import BoxError from '../boxerror.js';
import constants from '../constants.js';
import debugModule from 'debug';
import logger from '../logger.js';
import dig from '../dig.js';
import dns from '../dns.js';
import promiseRetry from '../promise-retry.js';
@@ -9,7 +9,7 @@ import safe from 'safetydance';
import superagent from '@cloudron/superagent';
import waitForDns from './waitfordns.js';
const debug = debugModule('box:dns/hetznercloud');
const { log, trace } = logger('dns/hetznercloud');
// https://docs.hetzner.cloud/reference/cloud
@@ -68,7 +68,7 @@ async function getRecords(domainConfig, zone, name, type) {
assert.strictEqual(typeof name, 'string');
assert.strictEqual(typeof type, 'string');
debug(`getRecords: getting dns records of ${zone.name} with ${name} and type ${type}`);
log(`getRecords: getting dns records of ${zone.name} with ${name} and type ${type}`);
const [error, response] = await safe(superagent.get(`${ENDPOINT}/zones/${zone.id}/rrsets/${name}/${type.toUpperCase()}`)
.set('Authorization', `Bearer ${domainConfig.token}`)
@@ -88,7 +88,7 @@ async function waitForAction(domainConfig, id) {
assert.strictEqual(typeof domainConfig, 'object');
assert.strictEqual(typeof id, 'number');
await promiseRetry({ times: 100, interval: 1000, debug }, async () => {
await promiseRetry({ times: 100, interval: 1000, debug: log }, async () => {
const [error, response] = await safe(superagent.get(`${ENDPOINT}/actions/${id}`)
.set('Authorization', `Bearer ${domainConfig.token}`)
.timeout(30 * 1000)
@@ -141,7 +141,7 @@ async function upsert(domainObject, location, type, values) {
zoneName = domainObject.zoneName,
name = dns.getName(domainObject, location, type) || '@';
debug(`upsert: ${name} for zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
log(`upsert: ${name} for zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
const zone = await getZone(domainConfig, zoneName);
const records = await getRecords(domainConfig, zone, name, type);
@@ -213,17 +213,17 @@ async function verifyDomainConfig(domainObject) {
// https://docs.hetzner.com/dns-console/dns/general/dns-overview#the-hetzner-online-name-servers-are
if (!nameservers.every(function (n) { return n.toLowerCase().search(/hetzner|your-server|second-ns/) !== -1; })) {
debug('verifyDomainConfig: %j does not contain Hetzner NS', nameservers);
log('verifyDomainConfig: %j does not contain Hetzner NS', nameservers);
if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Hetzner');
}
const location = 'cloudrontestdns';
await upsert(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record added');
log('verifyDomainConfig: Test A record added');
await del(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record removed again');
log('verifyDomainConfig: Test A record removed again');
return credentials;
}

18
src/dns/inwx.js
View File

@@ -2,13 +2,13 @@ import { ApiClient, Language } from 'domrobot-client';
import assert from 'node:assert';
import BoxError from '../boxerror.js';
import constants from '../constants.js';
import debugModule from 'debug';
import logger from '../logger.js';
import dig from '../dig.js';
import dns from '../dns.js';
import safe from 'safetydance';
import waitForDns from './waitfordns.js';
const debug = debugModule('box:dns/inwx');
const { log, trace } = logger('dns/inwx');
function formatError(response) {
@@ -49,7 +49,7 @@ async function getDnsRecords(domainConfig, apiClient, zoneName, fqdn, type) {
assert.strictEqual(typeof fqdn, 'string');
assert.strictEqual(typeof type, 'string');
debug(`getDnsRecords: ${fqdn} in zone ${zoneName} of type ${type}`);
log(`getDnsRecords: ${fqdn} in zone ${zoneName} of type ${type}`);
const [error, response] = await safe(apiClient.callApi('nameserver.info', { domain: zoneName, name: fqdn, type }));
if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
@@ -67,7 +67,7 @@ async function upsert(domainObject, location, type, values) {
const domainConfig = domainObject.config,
zoneName = domainObject.zoneName;
debug(`upsert: ${location} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
log(`upsert: ${location} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
const apiClient = await login(domainConfig);
const fqdn = dns.fqdn(location, domainObject.domain);
@@ -140,7 +140,7 @@ async function del(domainObject, location, type, values) {
const domainConfig = domainObject.config,
zoneName = domainObject.zoneName;
debug(`del: ${location} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
log(`del: ${location} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
const apiClient = await login(domainConfig);
const fqdn = dns.fqdn(location, domainObject.domain);
@@ -148,7 +148,7 @@ async function del(domainObject, location, type, values) {
if (result.length === 0) return;
const tmp = result.filter(function (record) { return values.some(function (value) { return value === record.content; }); });
debug('del: %j', tmp);
log('del: %j', tmp);
for (const r of tmp) {
const [error, response] = await safe(apiClient.callApi('nameserver.deleteRecord', { id: r.id }));
@@ -194,17 +194,17 @@ async function verifyDomainConfig(domainObject) {
if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers');
if (!nameservers.every(function (n) { return n.toLowerCase().search(/inwx|xnameserver|domrobot/) !== -1; })) {
debug('verifyDomainConfig: %j does not contain INWX NS', nameservers);
log('verifyDomainConfig: %j does not contain INWX NS', nameservers);
if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to INWX');
}
const location = 'cloudrontestdns';
await upsert(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record added');
log('verifyDomainConfig: Test A record added');
await del(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record removed again');
log('verifyDomainConfig: Test A record removed again');
return credentials;
}

18
src/dns/linode.js
View File

@@ -1,14 +1,14 @@
import assert from 'node:assert';
import constants from '../constants.js';
import BoxError from '../boxerror.js';
import debugModule from 'debug';
import logger from '../logger.js';
import dig from '../dig.js';
import dns from '../dns.js';
import safe from 'safetydance';
import superagent from '@cloudron/superagent';
import waitForDns from './waitfordns.js';
const debug = debugModule('box:dns/linode');
const { log, trace } = logger('dns/linode');
const LINODE_ENDPOINT = 'https://api.linode.com/v4';
@@ -47,7 +47,7 @@ async function getZoneId(domainConfig, zoneName) {
if (!zone || !zone.id) throw new BoxError(BoxError.NOT_FOUND, 'Zone not found');
debug(`getZoneId: zone id of ${zoneName} is ${zone.id}`);
log(`getZoneId: zone id of ${zoneName} is ${zone.id}`);
return zone.id;
}
@@ -58,7 +58,7 @@ async function getZoneRecords(domainConfig, zoneName, name, type) {
assert.strictEqual(typeof name, 'string');
assert.strictEqual(typeof type, 'string');
debug(`getInternal: getting dns records of ${zoneName} with ${name} and type ${type}`);
log(`getInternal: getting dns records of ${zoneName} with ${name} and type ${type}`);
const zoneId = await getZoneId(domainConfig, zoneName);
@@ -113,7 +113,7 @@ async function upsert(domainObject, location, type, values) {
zoneName = domainObject.zoneName,
name = dns.getName(domainObject, location, type) || '';
debug('upsert: %s for zone %s of type %s with values %j', name, zoneName, type, values);
log('upsert: %s for zone %s of type %s with values %j', name, zoneName, type, values);
const { zoneId, records } = await getZoneRecords(domainConfig, zoneName, name, type);
let i = 0; // used to track available records to update instead of create
@@ -176,7 +176,7 @@ async function upsert(domainObject, location, type, values) {
.retry(5)
.ok(() => true));
if (error) debug(`upsert: error removing record ${records[j].id}: ${error.message}`);
if (error) log(`upsert: error removing record ${records[j].id}: ${error.message}`);
}
}
@@ -244,17 +244,17 @@ async function verifyDomainConfig(domainObject) {
if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers');
if (nameservers.map(function (n) { return n.toLowerCase(); }).indexOf('ns1.linode.com') === -1) {
debug('verifyDomainConfig: %j does not contains linode NS', nameservers);
log('verifyDomainConfig: %j does not contains linode NS', nameservers);
if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Linode');
}
const location = 'cloudrontestdns';
await upsert(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record added');
log('verifyDomainConfig: Test A record added');
await del(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record removed again');
log('verifyDomainConfig: Test A record removed again');
return credentials;
}

6
src/dns/manual.js
View File

@@ -1,12 +1,12 @@
import assert from 'node:assert';
import BoxError from '../boxerror.js';
import debugModule from 'debug';
import logger from '../logger.js';
import dig from '../dig.js';
import dns from '../dns.js';
import safe from 'safetydance';
import waitForDns from './waitfordns.js';
const debug = debugModule('box:dns/manual');
const { log, trace } = logger('dns/manual');
function removePrivateFields(domainObject) {
@@ -24,7 +24,7 @@ async function upsert(domainObject, location, type, values) {
assert.strictEqual(typeof type, 'string');
assert(Array.isArray(values));
debug('upsert: %s for zone %s of type %s with values %j', location, domainObject.zoneName, type, values);
log('upsert: %s for zone %s of type %s with values %j', location, domainObject.zoneName, type, values);
return;
}

14
src/dns/namecheap.js
View File

@@ -1,7 +1,7 @@
import assert from 'node:assert';
import BoxError from '../boxerror.js';
import constants from '../constants.js';
import debugModule from 'debug';
import logger from '../logger.js';
import dig from '../dig.js';
import dns from '../dns.js';
import network from '../network.js';
@@ -12,7 +12,7 @@ import util from 'node:util';
import waitForDns from './waitfordns.js';
import xml2js from 'xml2js';
const debug = debugModule('box:dns/namecheap');
const { log, trace } = logger('dns/namecheap');
const ENDPOINT = 'https://api.namecheap.com/xml.response';
@@ -130,7 +130,7 @@ async function upsert(domainObject, subdomain, type, values) {
subdomain = dns.getName(domainObject, subdomain, type) || '@';
debug('upsert: %s for zone %s of type %s with values %j', subdomain, zoneName, type, values);
log('upsert: %s for zone %s of type %s with values %j', subdomain, zoneName, type, values);
const result = await getZone(domainConfig, zoneName);
@@ -210,7 +210,7 @@ async function del(domainObject, subdomain, type, values) {
subdomain = dns.getName(domainObject, subdomain, type) || '@';
debug('del: %s for zone %s of type %s with values %j', subdomain, zoneName, type, values);
log('del: %s for zone %s of type %s with values %j', subdomain, zoneName, type, values);
let result = await getZone(domainConfig, zoneName);
if (result.length === 0) return;
@@ -261,17 +261,17 @@ async function verifyDomainConfig(domainObject) {
if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers');
if (nameservers.some(function (n) { return n.toLowerCase().indexOf('.registrar-servers.com') === -1; })) {
debug('verifyDomainConfig: %j does not contains NC NS', nameservers);
log('verifyDomainConfig: %j does not contains NC NS', nameservers);
if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to NameCheap');
}
const testSubdomain = 'cloudrontestdns';
await upsert(domainObject, testSubdomain, 'A', [ip]);
debug('verifyDomainConfig: Test A record added');
log('verifyDomainConfig: Test A record added');
await del(domainObject, testSubdomain, 'A', [ip]);
debug('verifyDomainConfig: Test A record removed again');
log('verifyDomainConfig: Test A record removed again');
return credentials;
}

20
src/dns/namecom.js
View File

@@ -1,14 +1,14 @@
import assert from 'node:assert';
import BoxError from '../boxerror.js';
import constants from '../constants.js';
import debugModule from 'debug';
import logger from '../logger.js';
import dig from '../dig.js';
import dns from '../dns.js';
import safe from 'safetydance';
import superagent from '@cloudron/superagent';
import waitForDns from './waitfordns.js';
const debug = debugModule('box:dns/namecom');
const { log, trace } = logger('dns/namecom');
const NAMECOM_API = 'https://api.name.com/v4';
@@ -33,7 +33,7 @@ async function addRecord(domainConfig, zoneName, name, type, values) {
assert.strictEqual(typeof type, 'string');
assert(Array.isArray(values));
debug(`add: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
log(`add: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
const data = {
host: name,
@@ -71,7 +71,7 @@ async function updateRecord(domainConfig, zoneName, recordId, name, type, values
assert.strictEqual(typeof type, 'string');
assert(Array.isArray(values));
debug(`update:${recordId} on ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
log(`update:${recordId} on ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
const data = {
host: name,
@@ -107,7 +107,7 @@ async function getInternal(domainConfig, zoneName, name, type) {
assert.strictEqual(typeof name, 'string');
assert.strictEqual(typeof type, 'string');
debug(`getInternal: ${name} in zone ${zoneName} of type ${type}`);
log(`getInternal: ${name} in zone ${zoneName} of type ${type}`);
const [error, response] = await safe(superagent.get(`${NAMECOM_API}/domains/${zoneName}/records`)
.auth(domainConfig.username, domainConfig.token)
@@ -144,7 +144,7 @@ async function upsert(domainObject, location, type, values) {
zoneName = domainObject.zoneName,
name = dns.getName(domainObject, location, type) || '';
debug(`upsert: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
log(`upsert: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
const result = await getInternal(domainConfig, zoneName, name, type);
if (result.length === 0) return await addRecord(domainConfig, zoneName, name, type, values);
@@ -176,7 +176,7 @@ async function del(domainObject, location, type, values) {
zoneName = domainObject.zoneName,
name = dns.getName(domainObject, location, type) || '';
debug(`del: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
log(`del: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
const result = await getInternal(domainConfig, zoneName, name, type);
if (result.length === 0) return;
@@ -227,17 +227,17 @@ async function verifyDomainConfig(domainObject) {
if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers');
if (!nameservers.every(function (n) { return n.toLowerCase().indexOf('.name.com') !== -1; })) {
debug('verifyDomainConfig: %j does not contain Name.com NS', nameservers);
log('verifyDomainConfig: %j does not contain Name.com NS', nameservers);
if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to name.com');
}
const location = 'cloudrontestdns';
await upsert(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record added');
log('verifyDomainConfig: Test A record added');
await del(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record removed again');
log('verifyDomainConfig: Test A record removed again');
return credentials;
}

18
src/dns/netcup.js
View File

@@ -1,14 +1,14 @@
import assert from 'node:assert';
import BoxError from '../boxerror.js';
import constants from '../constants.js';
import debugModule from 'debug';
import logger from '../logger.js';
import dig from '../dig.js';
import dns from '../dns.js';
import safe from 'safetydance';
import superagent from '@cloudron/superagent';
import waitForDns from './waitfordns.js';
const debug = debugModule('box:dns/netcup');
const { log, trace } = logger('dns/netcup');
const API_ENDPOINT = 'https://ccp.netcup.net/run/webservice/servers/endpoint.php?JSON';
@@ -53,7 +53,7 @@ async function getAllRecords(domainConfig, apiSessionId, zoneName) {
assert.strictEqual(typeof apiSessionId, 'string');
assert.strictEqual(typeof zoneName, 'string');
debug(`getAllRecords: getting dns records of ${zoneName}`);
log(`getAllRecords: getting dns records of ${zoneName}`);
const data = {
action: 'infoDnsRecords',
@@ -82,7 +82,7 @@ async function upsert(domainObject, location, type, values) {
zoneName = domainObject.zoneName,
name = dns.getName(domainObject, location, type) || '@';
debug('upsert: %s for zone %s of type %s with values %j', name, zoneName, type, values);
log('upsert: %s for zone %s of type %s with values %j', name, zoneName, type, values);
const apiSessionId = await login(domainConfig);
@@ -138,7 +138,7 @@ async function get(domainObject, location, type) {
zoneName = domainObject.zoneName,
name = dns.getName(domainObject, location, type) || '@';
debug('get: %s for zone %s of type %s', name, zoneName, type);
log('get: %s for zone %s of type %s', name, zoneName, type);
const apiSessionId = await login(domainConfig);
@@ -158,7 +158,7 @@ async function del(domainObject, location, type, values) {
zoneName = domainObject.zoneName,
name = dns.getName(domainObject, location, type) || '@';
debug('del: %s for zone %s of type %s with values %j', name, zoneName, type, values);
log('del: %s for zone %s of type %s with values %j', name, zoneName, type, values);
const apiSessionId = await login(domainConfig);
@@ -239,17 +239,17 @@ async function verifyDomainConfig(domainObject) {
if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers');
if (!nameservers.every(function (n) { return n.toLowerCase().indexOf('dns.netcup.net') !== -1; })) {
debug('verifyDomainConfig: %j does not contains Netcup NS', nameservers);
log('verifyDomainConfig: %j does not contains Netcup NS', nameservers);
if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Netcup');
}
const location = 'cloudrontestdns';
await upsert(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record added');
log('verifyDomainConfig: Test A record added');
await del(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record removed again');
log('verifyDomainConfig: Test A record removed again');
return credentials;
}

6
src/dns/noop.js
View File

@@ -1,7 +1,7 @@
import assert from 'node:assert';
import debugModule from 'debug';
import logger from '../logger.js';
const debug = debugModule('box:dns/noop');
const { log, trace } = logger('dns/noop');
function removePrivateFields(domainObject) {
@@ -18,7 +18,7 @@ async function upsert(domainObject, location, type, values) {
assert.strictEqual(typeof type, 'string');
assert(Array.isArray(values));
debug('upsert: %s for zone %s of type %s with values %j', location, domainObject.zoneName, type, values);
log('upsert: %s for zone %s of type %s with values %j', location, domainObject.zoneName, type, values);
return;
}

20
src/dns/ovh.js
View File

@@ -1,14 +1,14 @@
import assert from 'node:assert';
import BoxError from '../boxerror.js';
import constants from '../constants.js';
import debugModule from 'debug';
import logger from '../logger.js';
import dig from '../dig.js';
import dns from '../dns.js';
import ovhClient from 'ovh';
import safe from 'safetydance';
import waitForDns from './waitfordns.js';
const debug = debugModule('box:dns/ovh');
const { log, trace } = logger('dns/ovh');
function formatError(error) {
@@ -39,7 +39,7 @@ async function getDnsRecordIds(domainConfig, zoneName, name, type) {
assert.strictEqual(typeof name, 'string');
assert.strictEqual(typeof type, 'string');
debug(`get: ${name} in zone ${zoneName} of type ${type}`);
log(`get: ${name} in zone ${zoneName} of type ${type}`);
const client = createClient(domainConfig);
const [error, data] = await safe(client.requestPromised('GET', `/domain/zone/${zoneName}/record`, { fieldType: type, subDomain: name }));
@@ -54,7 +54,7 @@ async function refreshZone(domainConfig, zoneName) {
assert.strictEqual(typeof domainConfig, 'object');
assert.strictEqual(typeof zoneName, 'string');
debug(`refresh: zone ${zoneName}`);
log(`refresh: zone ${zoneName}`);
const client = createClient(domainConfig);
const [error] = await safe(client.requestPromised('POST', `/domain/zone/${zoneName}/refresh`));
@@ -74,7 +74,7 @@ async function upsert(domainObject, location, type, values) {
zoneName = domainObject.zoneName,
name = dns.getName(domainObject, location, type) || '';
debug(`upsert: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
log(`upsert: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
const recordIds = await getDnsRecordIds(domainConfig, zoneName, name, type);
@@ -115,7 +115,7 @@ async function upsert(domainObject, location, type, values) {
}
await refreshZone(domainConfig, zoneName);
debug('upsert: completed');
log('upsert: completed');
}
async function get(domainObject, location, type) {
@@ -152,7 +152,7 @@ async function del(domainObject, location, type, values) {
zoneName = domainObject.zoneName,
name = dns.getName(domainObject, location, type) || '';
debug(`del: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
log(`del: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
const recordIds = await getDnsRecordIds(domainConfig, zoneName, name, type);
@@ -211,17 +211,17 @@ async function verifyDomainConfig(domainObject) {
// ovh.net, ovh.ca or anycast.me
if (!nameservers.every(function (n) { return n.toLowerCase().search(/ovh|kimsufi|anycast/) !== -1; })) {
debug('verifyDomainConfig: %j does not contain OVH NS', nameservers);
log('verifyDomainConfig: %j does not contain OVH NS', nameservers);
if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to OVH');
}
const location = 'cloudrontestdns';
await upsert(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record added');
log('verifyDomainConfig: Test A record added');
await del(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record removed again');
log('verifyDomainConfig: Test A record removed again');
return credentials;
}

18
src/dns/porkbun.js
View File

@@ -1,7 +1,7 @@
import assert from 'node:assert';
import BoxError from '../boxerror.js';
import constants from '../constants.js';
import debugModule from 'debug';
import logger from '../logger.js';
import dig from '../dig.js';
import dns from '../dns.js';
import safe from 'safetydance';
@@ -9,7 +9,7 @@ import superagent from '@cloudron/superagent';
import timers from 'timers/promises';
import waitForDns from './waitfordns.js';
const debug = debugModule('box:dns/porkbun');
const { log, trace } = logger('dns/porkbun');
// Rate limit note: Porkbun return 503 when it hits rate limits. It's as low as 1 req/second
@@ -44,7 +44,7 @@ async function getDnsRecords(domainConfig, zoneName, name, type) {
assert.strictEqual(typeof name, 'string');
assert.strictEqual(typeof type, 'string');
debug(`get: ${name} zone:${zoneName} type:${type}`);
log(`get: ${name} zone:${zoneName} type:${type}`);
const data = {
secretapikey: domainConfig.secretapikey,
@@ -90,7 +90,7 @@ async function upsert(domainObject, location, type, values) {
zoneName = domainObject.zoneName,
name = dns.getName(domainObject, location, type) || '';
debug(`upsert: ${name} zone:${zoneName} type:${type} values:${JSON.stringify(values)}`);
log(`upsert: ${name} zone:${zoneName} type:${type} values:${JSON.stringify(values)}`);
await delDnsRecords(domainConfig, zoneName, name, type);
@@ -119,7 +119,7 @@ async function upsert(domainObject, location, type, values) {
if (response.body.status !== 'SUCCESS') throw new BoxError(BoxError.EXTERNAL_ERROR, `Invalid status in response: ${JSON.stringify(response.body)}`);
if (!response.body.id) throw new BoxError(BoxError.EXTERNAL_ERROR, `Invalid id in response: ${JSON.stringify(response.body)}`);
debug(`upsert: created record with id ${response.body.id}`);
log(`upsert: created record with id ${response.body.id}`);
}
}
@@ -146,7 +146,7 @@ async function del(domainObject, location, type, values) {
zoneName = domainObject.zoneName,
name = dns.getName(domainObject, location, type) || '';
debug(`del: ${name} zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
log(`del: ${name} zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
const data = {
secretapikey: domainConfig.secretapikey,
@@ -203,17 +203,17 @@ async function verifyDomainConfig(domainObject) {
if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers');
if (!nameservers.every(function (n) { return n.toLowerCase().indexOf('.ns.porkbun.com') !== -1; })) {
debug('verifyDomainConfig: %j does not contain Porkbun NS', nameservers);
log('verifyDomainConfig: %j does not contain Porkbun NS', nameservers);
if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Porkbun');
}
const location = 'cloudrontestdns';
await upsert(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record added');
log('verifyDomainConfig: Test A record added');
await del(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record removed again');
log('verifyDomainConfig: Test A record removed again');
return credentials;
}

14
src/dns/route53.js
View File

@@ -2,7 +2,7 @@ import assert from 'node:assert';
import BoxError from '../boxerror.js';
import { ConfiguredRetryStrategy } from '@smithy/util-retry';
import constants from '../constants.js';
import debugModule from 'debug';
import logger from '../logger.js';
import dig from '../dig.js';
import dns from '../dns.js';
import { Route53 } from '@aws-sdk/client-route-53';
@@ -10,7 +10,7 @@ import safe from 'safetydance';
import waitForDns from './waitfordns.js';
import _ from '../underscore.js';
const debug = debugModule('box:dns/route53');
const { log, trace } = logger('dns/route53');
function removePrivateFields(domainObject) {
@@ -93,7 +93,7 @@ async function upsert(domainObject, location, type, values) {
zoneName = domainObject.zoneName,
fqdn = dns.fqdn(location, domainObject.domain);
debug('add: %s for zone %s of type %s with values %j', fqdn, zoneName, type, values);
log('add: %s for zone %s of type %s with values %j', fqdn, zoneName, type, values);
const zone = await getZoneByName(domainConfig, zoneName);
@@ -243,7 +243,7 @@ async function verifyDomainConfig(domainObject) {
const zone = await getHostedZone(credentials, zoneName);
if (!_.isEqual(zone.DelegationSet.NameServers.sort(), nameservers.sort())) {
debug('verifyDomainConfig: %j and %j do not match', nameservers, zone.DelegationSet.NameServers);
log('verifyDomainConfig: %j and %j do not match', nameservers, zone.DelegationSet.NameServers);
if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Route53');
}
@@ -251,13 +251,13 @@ async function verifyDomainConfig(domainObject) {
const newDomainObject = Object.assign({ }, domainObject, { config: credentials });
await upsert(newDomainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record added');
log('verifyDomainConfig: Test A record added');
await get(newDomainObject, location, 'A');
debug('verifyDomainConfig: Can list record sets');
log('verifyDomainConfig: Can list record sets');
await del(newDomainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record removed again');
log('verifyDomainConfig: Test A record removed again');
return credentials;
}

18
src/dns/vultr.js
View File

@@ -1,14 +1,14 @@
import assert from 'node:assert';
import constants from '../constants.js';
import BoxError from '../boxerror.js';
import debugModule from 'debug';
import logger from '../logger.js';
import dig from '../dig.js';
import dns from '../dns.js';
import safe from 'safetydance';
import superagent from '@cloudron/superagent';
import waitForDns from './waitfordns.js';
const debug = debugModule('box:dns/vultr');
const { log, trace } = logger('dns/vultr');
const VULTR_ENDPOINT = 'https://api.vultr.com/v2';
@@ -32,7 +32,7 @@ async function getZoneRecords(domainConfig, zoneName, name, type) {
assert.strictEqual(typeof name, 'string');
assert.strictEqual(typeof type, 'string');
debug(`getInternal: getting dns records of ${zoneName} with ${name} and type ${type}`);
log(`getInternal: getting dns records of ${zoneName} with ${name} and type ${type}`);
const per_page = 100;
let cursor = null, records = [];
@@ -80,7 +80,7 @@ async function upsert(domainObject, location, type, values) {
zoneName = domainObject.zoneName,
name = dns.getName(domainObject, location, type) || '';
debug('upsert: %s for zone %s of type %s with values %j', name, zoneName, type, values);
log('upsert: %s for zone %s of type %s with values %j', name, zoneName, type, values);
const records = await getZoneRecords(domainConfig, zoneName, name, type);
@@ -143,10 +143,10 @@ async function upsert(domainObject, location, type, values) {
.timeout(30 * 1000)
.retry(5));
if (error) debug(`upsert: error removing record ${records[j].id}: ${error.message}`);
if (error) log(`upsert: error removing record ${records[j].id}: ${error.message}`);
}
debug('upsert: completed with recordIds:%j', recordIds);
log('upsert: completed with recordIds:%j', recordIds);
}
async function del(domainObject, location, type, values) {
@@ -214,17 +214,17 @@ async function verifyDomainConfig(domainObject) {
if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers');
if (nameservers.map(function (n) { return n.toLowerCase(); }).indexOf('ns1.vultr.com') === -1) {
debug('verifyDomainConfig: %j does not contains vultr NS', nameservers);
log('verifyDomainConfig: %j does not contains vultr NS', nameservers);
if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Vultr');
}
const location = 'cloudrontestdns';
await upsert(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record added');
log('verifyDomainConfig: Test A record added');
await del(domainObject, location, 'A', [ ip ]);
debug('verifyDomainConfig: Test A record removed again');
log('verifyDomainConfig: Test A record removed again');
return credentials;
}

28
src/dns/waitfordns.js
View File

@@ -1,13 +1,13 @@
import assert from 'node:assert';
import BoxError from '../boxerror.js';
import debugModule from 'debug';
import logger from '../logger.js';
import dig from '../dig.js';
import dns from 'node:dns';
import promiseRetry from '../promise-retry.js';
import safe from 'safetydance';
import _ from '../underscore.js';
const debug = debugModule('box:dns/waitfordns');
const { log, trace } = logger('dns/waitfordns');
async function resolveIp(hostname, type, options) {
assert.strictEqual(typeof hostname, 'string');
@@ -15,17 +15,17 @@ async function resolveIp(hostname, type, options) {
assert.strictEqual(typeof options, 'object');
// try A record at authoritative server
debug(`resolveIp: Checking ${type} for ${hostname} at ${options.server}`);
log(`resolveIp: Checking ${type} for ${hostname} at ${options.server}`);
const [error, results] = await safe(dig.resolve(hostname, type, options));
if (!error && results.length !== 0) return results;
// try CNAME record at authoritative server
debug(`resolveIp: No ${type}. Checking CNAME for ${hostname} at ${options.server}`);
log(`resolveIp: No ${type}. Checking CNAME for ${hostname} at ${options.server}`);
const cnameResults = await dig.resolve(hostname, 'CNAME', options);
if (cnameResults.length === 0) return cnameResults;
// recurse lookup the CNAME record
debug(`resolveIp: found CNAME for ${hostname}. resolving ${cnameResults[0]}`);
log(`resolveIp: found CNAME for ${hostname}. resolving ${cnameResults[0]}`);
return await dig.resolve(cnameResults[0], type, _.omit(options, ['server']));
}
@@ -40,7 +40,7 @@ async function isChangeSynced(hostname, type, value, nameserver) {
const [error6, nsIPv6s] = await safe(dig.resolve(nameserver, 'AAAA', { timeout: 5000 }));
if (error4 && error6) {
debug(`isChangeSynced: cannot resolve NS ${nameserver}`); // NS doesn't resolve at all; it's fine
log(`isChangeSynced: cannot resolve NS ${nameserver}`); // NS doesn't resolve at all; it's fine
return true;
}
@@ -54,13 +54,13 @@ async function isChangeSynced(hostname, type, value, nameserver) {
const [error, answer] = await safe(resolver);
// CONNREFUSED - when there is no ipv4/ipv6 connectivity. REFUSED - server won't answer maybe by policy
if (error && (error.code === dns.TIMEOUT || error.code === dns.REFUSED || error.code === dns.CONNREFUSED)) {
debug(`isChangeSynced: NS ${nameserver} (${nsIp}) not resolving ${hostname} (${type}): ${error}. Ignoring`);
log(`isChangeSynced: NS ${nameserver} (${nsIp}) not resolving ${hostname} (${type}): ${error}. Ignoring`);
status[i] = true; // should be ok if dns server is down
continue;
}
if (error) {
debug(`isChangeSynced: NS ${nameserver} (${nsIp}) errored when resolve ${hostname} (${type}): ${error}`);
log(`isChangeSynced: NS ${nameserver} (${nsIp}) errored when resolve ${hostname} (${type}): ${error}`);
status[i] = false;
continue;
}
@@ -72,7 +72,7 @@ async function isChangeSynced(hostname, type, value, nameserver) {
match = answer.some(function (a) { return value === a.join(''); });
}
debug(`isChangeSynced: ${hostname} (${type}) was resolved to ${answer} at NS ${nameserver} (${nsIp}). Expecting ${value}. Match ${match}`);
log(`isChangeSynced: ${hostname} (${type}) was resolved to ${answer} at NS ${nameserver} (${nsIp}). Expecting ${value}. Match ${match}`);
status[i] = match;
}
@@ -87,21 +87,21 @@ async function waitForDns(hostname, zoneName, type, value, options) {
assert.strictEqual(typeof value, 'string');
assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
debug(`waitForDns: waiting for ${hostname} to be ${value} in zone ${zoneName}`);
log(`waitForDns: waiting for ${hostname} to be ${value} in zone ${zoneName}`);
await promiseRetry(Object.assign({ debug }, options), async function () {
await promiseRetry(Object.assign({ debug: log }, options), async function () {
const nameservers = await dig.resolve(zoneName, 'NS', { timeout: 5000 });
if (!nameservers) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Unable to get nameservers');
debug(`waitForDns: nameservers are ${JSON.stringify(nameservers)}`);
log(`waitForDns: nameservers are ${JSON.stringify(nameservers)}`);
for (const nameserver of nameservers) {
const synced = await isChangeSynced(hostname, type, value, nameserver);
debug(`waitForDns: ${hostname} at ns ${nameserver}: ${synced ? 'done' : 'not done'} `);
log(`waitForDns: ${hostname} at ns ${nameserver}: ${synced ? 'done' : 'not done'} `);
if (!synced) throw new BoxError(BoxError.EXTERNAL_ERROR, 'ETRYAGAIN');
}
});
debug(`waitForDns: ${hostname} has propagated`);
log(`waitForDns: ${hostname} has propagated`);
}
export default waitForDns;

6
src/dns/wildcard.js
View File

@@ -1,13 +1,13 @@
import assert from 'node:assert';
import BoxError from '../boxerror.js';
import debugModule from 'debug';
import logger from '../logger.js';
import dig from '../dig.js';
import dns from '../dns.js';
import network from '../network.js';
import safe from 'safetydance';
import waitForDns from './waitfordns.js';
const debug = debugModule('box:dns/manual');
const { log, trace } = logger('dns/manual');
function removePrivateFields(domainObject) {
@@ -24,7 +24,7 @@ async function upsert(domainObject, location, type, values) {
assert.strictEqual(typeof type, 'string');
assert(Array.isArray(values));
debug('upsert: %s for zone %s of type %s with values %j', location, domainObject.zoneName, type, values);
log('upsert: %s for zone %s of type %s with values %j', location, domainObject.zoneName, type, values);
return;
}

46
src/docker.js
View File

@@ -3,7 +3,7 @@ import assert from 'node:assert';
import BoxError from './boxerror.js';
import constants from './constants.js';
import dashboard from './dashboard.js';
import debugModule from 'debug';
import logger from './logger.js';
import Docker from 'dockerode';
import dockerRegistries from './dockerregistries.js';
import fs from 'node:fs';
@@ -17,7 +17,7 @@ import safe from 'safetydance';
import timers from 'timers/promises';
import volumes from './volumes.js';
const debug = debugModule('box:docker');
const { log, trace } = logger('docker');
const shell = shellModule('docker');
@@ -94,7 +94,7 @@ async function pullImage(imageRef) {
const authConfig = await getAuthConfig(imageRef);
debug(`pullImage: will pull ${imageRef}. auth: ${authConfig ? 'yes' : 'no'}`);
log(`pullImage: will pull ${imageRef}. auth: ${authConfig ? 'yes' : 'no'}`);
const [error, stream] = await safe(gConnection.pull(imageRef, { authconfig: authConfig }));
if (error && error.statusCode === 404) throw new BoxError(BoxError.NOT_FOUND, `Unable to pull image ${imageRef}. message: ${error.message} statusCode: ${error.statusCode}`);
@@ -107,17 +107,17 @@ async function pullImage(imageRef) {
let layerError = null;
stream.on('data', function (chunk) {
const data = safe.JSON.parse(chunk) || { };
debug('pullImage: %j', data);
log('pullImage: %j', data);
// The data.status here is useless because this is per layer as opposed to per image
if (!data.status && data.error) { // data is { errorDetail: { message: xx } , error: xx }
debug(`pullImage error ${imageRef}: ${data.errorDetail.message}`);
log(`pullImage error ${imageRef}: ${data.errorDetail.message}`);
layerError = data.errorDetail;
}
});
stream.on('end', function () {
debug(`downloaded image ${imageRef} . error: ${!!layerError}`);
log(`downloaded image ${imageRef} . error: ${!!layerError}`);
if (!layerError) return resolve();
@@ -125,7 +125,7 @@ async function pullImage(imageRef) {
});
stream.on('error', function (streamError) { // this is only hit for stream error and not for some download error
debug(`error pulling image ${imageRef}: %o`, streamError);
log(`error pulling image ${imageRef}: %o`, streamError);
reject(new BoxError(BoxError.DOCKER_ERROR, streamError.message));
});
});
@@ -135,7 +135,7 @@ async function buildImage(dockerImage, sourceArchiveFilePath) {
assert.strictEqual(typeof dockerImage, 'string');
assert.strictEqual(typeof sourceArchiveFilePath, 'string');
debug(`buildImage: building ${dockerImage} from ${sourceArchiveFilePath}`);
log(`buildImage: building ${dockerImage} from ${sourceArchiveFilePath}`);
const buildOptions = { t: dockerImage };
const [listError, listOut] = await safe(shell.spawn('tar', ['-tzf', sourceArchiveFilePath], { encoding: 'utf8' }));
@@ -146,7 +146,7 @@ async function buildImage(dockerImage, sourceArchiveFilePath) {
});
if (dockerfileCloudronPath) {
buildOptions.dockerfile = dockerfileCloudronPath.replace(/\/$/, '');
debug(`buildImage: using ${buildOptions.dockerfile}`);
log(`buildImage: using ${buildOptions.dockerfile}`);
}
}
@@ -164,23 +164,23 @@ async function buildImage(dockerImage, sourceArchiveFilePath) {
buildError = data.errorDetail || { message: data.error };
} else {
const message = (data.stream || data.status || data.aux?.ID || '').replace(/\n$/, '');
if (message) debug('buildImage: ' + message);
if (message) log('buildImage: ' + message);
}
});
stream.on('end', () => {
if (buildError) {
debug(`buildImage: error ${buildError}`);
log(`buildImage: error ${buildError}`);
return reject(new BoxError(buildError.message.includes('no space') ? BoxError.FS_ERROR : BoxError.DOCKER_ERROR, buildError.message));
} else {
debug(`buildImage: success ${dockerImage}`);
log(`buildImage: success ${dockerImage}`);
}
resolve();
});
stream.on('error', (streamError) => {
debug(`buildImage: error building image ${dockerImage}: %o`, streamError);
log(`buildImage: error building image ${dockerImage}: %o`, streamError);
reject(new BoxError(BoxError.DOCKER_ERROR, streamError.message));
});
});
@@ -329,7 +329,7 @@ async function restartContainer(containerId) {
async function stopContainer(containerId) {
assert.strictEqual(typeof containerId, 'string');
debug(`stopContainer: stopping container ${containerId}`);
log(`stopContainer: stopping container ${containerId}`);
const container = gConnection.getContainer(containerId);
@@ -347,7 +347,7 @@ async function stopContainer(containerId) {
async function deleteContainer(containerId) { // id can also be name
assert.strictEqual(typeof containerId, 'string');
debug(`deleteContainer: deleting ${containerId}`);
log(`deleteContainer: deleting ${containerId}`);
const container = gConnection.getContainer(containerId);
@@ -360,7 +360,7 @@ async function deleteContainer(containerId) { // id can also be name
if (error && error.statusCode === 404) return;
if (error) {
debug('Error removing container %s : %o', containerId, error);
log('Error removing container %s : %o', containerId, error);
throw new BoxError(BoxError.DOCKER_ERROR, error);
}
}
@@ -405,14 +405,14 @@ async function deleteImage(imageRef) {
// registry v1 used to pull down all *tags*. this meant that deleting image by tag was not enough (since that
// just removes the tag). we used to remove the image by id. this is not required anymore because aliases are
// not created anymore after https://github.com/docker/docker/pull/10571
debug(`deleteImage: removing ${imageRef}`);
log(`deleteImage: removing ${imageRef}`);
const [error] = await safe(gConnection.getImage(imageRef.replace(/@sha256:.*/,'')).remove(removeOptions)); // can't have the manifest id. won't remove anythin
if (error && error.statusCode === 400) return; // invalid image format. this can happen if user installed with a bad --docker-image
if (error && error.statusCode === 404) return; // not found
if (error && error.statusCode === 409) return; // another container using the image
if (error) {
debug(`Error removing image ${imageRef} : %o`, error);
log(`Error removing image ${imageRef} : %o`, error);
throw new BoxError(BoxError.DOCKER_ERROR, error);
}
}
@@ -432,7 +432,7 @@ async function inspect(containerId) {
async function downloadImage(manifest) {
assert.strictEqual(typeof manifest, 'object');
debug(`downloadImage: ${manifest.dockerImage}`);
log(`downloadImage: ${manifest.dockerImage}`);
const image = gConnection.getImage(manifest.dockerImage);
@@ -441,7 +441,7 @@ async function downloadImage(manifest) {
const parsedManifestRef = parseImageRef(manifest.dockerImage);
await promiseRetry({ times: 10, interval: 5000, debug, retry: (pullError) => pullError.reason !== BoxError.FS_ERROR }, async () => {
await promiseRetry({ times: 10, interval: 5000, debug: log, retry: (pullError) => pullError.reason !== BoxError.FS_ERROR }, async () => {
// custom (non appstore) image
if (parsedManifestRef.registry !== null || !parsedManifestRef.fullRepositoryName.startsWith('cloudron/')) return await pullImage(manifest.dockerImage);
@@ -457,9 +457,9 @@ async function downloadImage(manifest) {
if (pullError || !upstreamRef) throw new BoxError(BoxError.DOCKER_ERROR, `Unable to pull ${manifest.dockerImage} from dockerhub or quay: ${pullError?.message}`);
// retag the downloaded image to not have the registry name. this prevents 'docker run' from redownloading it
debug(`downloadImage: tagging ${upstreamRef} as ${parsedManifestRef.fullRepositoryName}:${parsedManifestRef.tag}`);
log(`downloadImage: tagging ${upstreamRef} as ${parsedManifestRef.fullRepositoryName}:${parsedManifestRef.tag}`);
await gConnection.getImage(upstreamRef).tag({ repo: parsedManifestRef.fullRepositoryName, tag: parsedManifestRef.tag });
debug(`downloadImage: untagging ${upstreamRef}`);
log(`downloadImage: untagging ${upstreamRef}`);
await deleteImage(upstreamRef);
});
}
@@ -731,7 +731,7 @@ async function createSubcontainer(app, name, cmd, options) {
containerOptions.HostConfig.Devices = Object.keys(app.devices).map((d) => {
if (!safe.fs.existsSync(d)) {
debug(`createSubcontainer: device ${d} does not exist. Skipping...`);
log(`createSubcontainer: device ${d} does not exist. Skipping...`);
return null;
}

16
src/dockerproxy.js
View File

@@ -2,7 +2,7 @@ import apps from './apps.js';
import assert from 'node:assert';
import constants from './constants.js';
import express from 'express';
import debugModule from 'debug';
import logger from './logger.js';
import http from 'node:http';
import { HttpError } from '@cloudron/connect-lastmile';
import middleware from './middleware/index.js';
@@ -13,7 +13,7 @@ import safe from 'safetydance';
import util from 'node:util';
import volumes from './volumes.js';
const debug = debugModule('box:dockerproxy');
const { log, trace } = logger('dockerproxy');
let gHttpServer = null;
@@ -49,7 +49,7 @@ function attachDockerRequest(req, res, next) {
// Force node to send out the headers, this is required for the /container/wait api to make the docker cli proceed
res.write(' ');
dockerResponse.on('error', function (error) { debug('dockerResponse error: %o', error); });
dockerResponse.on('error', function (error) { log('dockerResponse error: %o', error); });
dockerResponse.pipe(res, { end: true });
});
@@ -66,7 +66,7 @@ async function containersCreate(req, res, next) {
const appDataDir = path.join(paths.APPS_DATA_DIR, req.resources.app.id, 'data');
debug('containersCreate: original bind mounts:', req.body.HostConfig.Binds);
log('containersCreate: original bind mounts:', req.body.HostConfig.Binds);
const [error, result] = await safe(volumes.list());
if (error) return next(new HttpError(500, `Error listing volumes: ${error.message}`));
@@ -82,14 +82,14 @@ async function containersCreate(req, res, next) {
const volumeName = bind.match(new RegExp('/media/([^:/]+)/?'))[1];
const volume = volumesByName[volumeName];
if (volume) binds.push(bind.replace(new RegExp(`^/media/${volumeName}`), volume.hostPath));
else debug(`containersCreate: dropped unknown volume ${volumeName}`);
else log(`containersCreate: dropped unknown volume ${volumeName}`);
} else {
req.dockerRequest.abort();
return next(new HttpError(400, 'Binds must be under /app/data/ or /media'));
}
}
debug('containersCreate: rewritten bind mounts:', binds);
log('containersCreate: rewritten bind mounts:', binds);
safe.set(req.body, 'HostConfig.Binds', binds);
const plainBody = JSON.stringify(req.body);
@@ -125,7 +125,7 @@ async function start() {
if (constants.TEST) {
proxyServer.use(function (req, res, next) {
debug('proxying: ' + req.method, req.url);
log('proxying: ' + req.method, req.url);
next();
});
}
@@ -177,7 +177,7 @@ async function start() {
});
});
debug(`start: listening on 172.18.0.1:${constants.DOCKER_PROXY_PORT}`);
log(`start: listening on 172.18.0.1:${constants.DOCKER_PROXY_PORT}`);
await util.promisify(gHttpServer.listen.bind(gHttpServer))(constants.DOCKER_PROXY_PORT, '172.18.0.1');
}

10
src/domains.js
View File

@@ -4,7 +4,7 @@ import constants from './constants.js';
import crypto from 'node:crypto';
import dashboard from './dashboard.js';
import database from './database.js';
import debugModule from 'debug';
import logger from './logger.js';
import eventlog from './eventlog.js';
import mailServer from './mailserver.js';
import notifications from './notifications.js';
@@ -36,7 +36,7 @@ import dnsManual from './dns/manual.js';
import dnsPorkbun from './dns/porkbun.js';
import dnsWildcard from './dns/wildcard.js';
const debug = debugModule('box:domains');
const { log, trace } = logger('domains');
const DOMAINS_FIELDS = [ 'domain', 'zoneName', 'provider', 'configJson', 'tlsConfigJson', 'wellKnownJson', 'fallbackCertificateJson' ].join(',');
@@ -187,7 +187,7 @@ async function add(domain, data, auditSource) {
await eventlog.add(eventlog.ACTION_DOMAIN_ADD, auditSource, { domain, zoneName, provider });
safe(mailServer.onDomainAdded(domain), { debug }); // background
safe(mailServer.onDomainAdded(domain), { debug: log }); // background
}
async function get(domain) {
@@ -341,7 +341,7 @@ async function getDomainObjectMap() {
async function checkConfigs(auditSource) {
assert.strictEqual(typeof auditSource, 'object');
debug(`checkConfig: validating domain configs`);
log(`checkConfig: validating domain configs`);
for (const domainObject of await list()) {
if (domainObject.provider === 'noop' || domainObject.provider === 'manual' || domainObject.provider === 'wildcard') {
@@ -366,7 +366,7 @@ async function checkConfigs(auditSource) {
errorMessage = `General error: ${error.message}`;
}
debug(`checkConfig: ${domainObject.domain} is not configured properly`, error);
log(`checkConfig: ${domainObject.domain} is not configured properly`, error);
await notifications.pin(notifications.TYPE_DOMAIN_CONFIG_CHECK_FAILED, `Domain ${domainObject.domain} is not configured properly`,
errorMessage, { context: domainObject.domain });

20
src/dyndns.js
View File

@@ -1,7 +1,7 @@
import apps from './apps.js';
import assert from 'node:assert';
import dashboard from './dashboard.js';
import debugModule from 'debug';
import logger from './logger.js';
import dns from './dns.js';
import eventlog from './eventlog.js';
import fs from 'node:fs';
@@ -11,7 +11,7 @@ import paths from './paths.js';
import safe from 'safetydance';
import tasks from './tasks.js';
const debug = debugModule('box:dyndns');
const { log, trace } = logger('dyndns');
// FIXME: this races with apptask. can result in a conflict if apptask is doing some dns operation and this code changes entries
@@ -26,11 +26,11 @@ async function refreshDns(auditSource) {
const ipv6Changed = ipv6 && info.ipv6 !== ipv6; // both should be RFC 5952 format
if (!ipv4Changed && !ipv6Changed) {
debug(`refreshDns: no change in IP. ipv4: ${ipv4} ipv6: ${ipv6}`);
log(`refreshDns: no change in IP. ipv4: ${ipv4} ipv6: ${ipv6}`);
return;
}
debug(`refreshDns: updating IP from ${info.ipv4} to ipv4: ${ipv4} (changed: ${ipv4Changed}) ipv6: ${ipv6} (changed: ${ipv6Changed})`);
log(`refreshDns: updating IP from ${info.ipv4} to ipv4: ${ipv4} (changed: ${ipv4Changed}) ipv6: ${ipv6} (changed: ${ipv6Changed})`);
const taskId = await tasks.add(tasks.TASK_SYNC_DYNDNS, [ ipv4Changed ? ipv4 : null, ipv6Changed ? ipv6 : null, auditSource ]);
// background
@@ -57,15 +57,15 @@ async function sync(ipv4, ipv6, auditSource, progressCallback) {
let percent = 5;
const { domain:dashboardDomain, fqdn:dashboardFqdn, subdomain:dashboardSubdomain } = await dashboard.getLocation();
progressCallback({ percent, message: `Updating dashboard location ${dashboardFqdn}`});
if (ipv4) await safe(dns.upsertDnsRecords(dashboardSubdomain, dashboardDomain, 'A', [ ipv4 ]), { debug });
if (ipv6) await safe(dns.upsertDnsRecords(dashboardSubdomain, dashboardDomain, 'AAAA', [ ipv6 ]), { debug });
if (ipv4) await safe(dns.upsertDnsRecords(dashboardSubdomain, dashboardDomain, 'A', [ ipv4 ]), { debug: log });
if (ipv6) await safe(dns.upsertDnsRecords(dashboardSubdomain, dashboardDomain, 'AAAA', [ ipv6 ]), { debug: log });
const { domain:mailDomain, fqdn:mailFqdn, subdomain:mailSubdomain } = await mailServer.getLocation();
percent += 10;
progressCallback({ percent, message: `Updating mail location ${mailFqdn}`});
if (dashboardFqdn !== mailFqdn) {
if (ipv4) await safe(dns.upsertDnsRecords(mailSubdomain, mailDomain, 'A', [ ipv4 ]), { debug });
if (ipv6) await safe(dns.upsertDnsRecords(mailSubdomain, mailDomain, 'AAAA', [ ipv6 ]), { debug });
if (ipv4) await safe(dns.upsertDnsRecords(mailSubdomain, mailDomain, 'A', [ ipv4 ]), { debug: log });
if (ipv6) await safe(dns.upsertDnsRecords(mailSubdomain, mailDomain, 'AAAA', [ ipv6 ]), { debug: log });
}
const result = await apps.list();
@@ -79,8 +79,8 @@ async function sync(ipv4, ipv6, auditSource, progressCallback) {
.concat(app.aliasDomains);
for (const location of locations) {
if (ipv4) await safe(dns.upsertDnsRecords(location.subdomain, location.domain, 'A', [ ipv4 ]), { debug });
if (ipv6) await safe(dns.upsertDnsRecords(location.subdomain, location.domain, 'AAAA', [ ipv6 ], { debug }));
if (ipv4) await safe(dns.upsertDnsRecords(location.subdomain, location.domain, 'A', [ ipv4 ]), { debug: log });
if (ipv6) await safe(dns.upsertDnsRecords(location.subdomain, location.domain, 'AAAA', [ ipv6 ], { debug: log }));
}
}

6
src/eventlog.js
View File

@@ -1,12 +1,12 @@
import assert from 'node:assert';
import crypto from 'node:crypto';
import database from './database.js';
import debugModule from 'debug';
import logger from './logger.js';
import mysql from 'mysql2';
import notifications from './notifications.js';
import safe from 'safetydance';
const debug = debugModule('box:eventlog');
const { log, trace } = logger('eventlog');
const ACTION_ACTIVATE = 'cloudron.activate';
const ACTION_USER_LOGIN = 'user.login';
@@ -119,7 +119,7 @@ async function cleanup(options) {
assert.strictEqual(typeof options, 'object');
const creationTime = options.creationTime;
debug(`cleanup: pruning events. creationTime: ${creationTime.toString()}`);
log(`cleanup: pruning events. creationTime: ${creationTime.toString()}`);
// only these actions are pruned
const actions = [

78
src/externalldap.js
View File

@@ -3,7 +3,7 @@ import AuditSource from './auditsource.js';
import BoxError from './boxerror.js';
import constants from './constants.js';
import cron from './cron.js';
import debugModule from 'debug';
import logger from './logger.js';
import eventlog from './eventlog.js';
import groups from './groups.js';
import ldap from 'ldapjs';
@@ -13,7 +13,7 @@ import tasks from './tasks.js';
import users from './users.js';
import util from 'node:util';
const debug = debugModule('box:externalldap');
const { log, trace } = logger('externalldap');
function removePrivateFields(ldapConfig) {
@@ -38,7 +38,7 @@ function translateUser(ldapConfig, ldapUser) {
};
if (!user.username || !user.email || !user.displayName) {
debug(`[Invalid LDAP user] username=${user.username} email=${user.email} displayName=${user.displayName}`);
log(`[Invalid LDAP user] username=${user.username} email=${user.email} displayName=${user.displayName}`);
return null;
}
@@ -79,7 +79,7 @@ async function getClient(config, options) {
return await new Promise((resolve, reject) => {
// ensure we don't just crash
client.on('error', function (error) { // don't reject, we must have gotten a bind error
debug('getClient: ExternalLdap client error:', error);
log('getClient: ExternalLdap client error:', error);
});
// skip bind auth if none exist or if not wanted
@@ -133,16 +133,16 @@ async function supportsPagination(client) {
const result = await clientSearch(client, '', searchOptions);
const controls = result.supportedControl;
if (!controls || !Array.isArray(controls)) {
debug('supportsPagination: no supportedControl attribute returned');
log('supportsPagination: no supportedControl attribute returned');
return false;
}
if (!controls.includes(ldap.PagedResultsControl.OID)) {
debug('supportsPagination: server does not support pagination. Available controls:', controls);
log('supportsPagination: server does not support pagination. Available controls:', controls);
return false;
}
debug('supportsPagination: server supports pagination');
log('supportsPagination: server supports pagination');
return true;
}
@@ -150,7 +150,7 @@ async function ldapGetByDN(config, dn) {
assert.strictEqual(typeof config, 'object');
assert.strictEqual(typeof dn, 'string');
debug(`ldapGetByDN: Get object at ${dn}`);
log(`ldapGetByDN: Get object at ${dn}`);
const client = await getClient(config, { bind: true });
const paged = await supportsPagination(client);
@@ -343,7 +343,7 @@ async function startSyncer() {
if (config.provider === 'noop') throw new BoxError(BoxError.BAD_STATE, 'not enabled');
const taskId = await tasks.add(tasks.TASK_SYNC_EXTERNAL_LDAP, []);
safe(tasks.startTask(taskId, {}), { debug }); // background
safe(tasks.startTask(taskId, {}), { debug: log }); // background
return taskId;
}
@@ -353,7 +353,7 @@ async function syncUsers(config, progressCallback) {
const ldapUsers = await ldapUserSearch(config, {});
debug(`syncUsers: Found ${ldapUsers.length} users`);
log(`syncUsers: Found ${ldapUsers.length} users`);
let percent = 10;
const step = 28 / (ldapUsers.length + 1);
@@ -369,27 +369,27 @@ async function syncUsers(config, progressCallback) {
const user = await users.getByUsername(ldapUser.username);
if (!user) {
debug(`syncUsers: [adding user] username=${ldapUser.username} email=${ldapUser.email} displayName=${ldapUser.displayName}`);
log(`syncUsers: [adding user] username=${ldapUser.username} email=${ldapUser.email} displayName=${ldapUser.displayName}`);
const [userAddError] = await safe(users.add(ldapUser.email, { username: ldapUser.username, password: null, displayName: ldapUser.displayName, source: 'ldap' }, AuditSource.EXTERNAL_LDAP));
if (userAddError) debug('syncUsers: Failed to create user. %j %o', ldapUser, userAddError);
if (userAddError) log('syncUsers: Failed to create user. %j %o', ldapUser, userAddError);
} else if (user.source !== 'ldap') {
debug(`syncUsers: [mapping user] username=${ldapUser.username} email=${ldapUser.email} displayName=${ldapUser.displayName}`);
log(`syncUsers: [mapping user] username=${ldapUser.username} email=${ldapUser.email} displayName=${ldapUser.displayName}`);
const [userMappingError] = await safe(users.update(user, { email: ldapUser.email, fallbackEmail: ldapUser.email, displayName: ldapUser.displayName, source: 'ldap' }, AuditSource.EXTERNAL_LDAP));
if (userMappingError) debug('Failed to map user. %j %o', ldapUser, userMappingError);
if (userMappingError) log('Failed to map user. %j %o', ldapUser, userMappingError);
} else if (user.email !== ldapUser.email || user.displayName !== ldapUser.displayName) {
debug(`syncUsers: [updating user] username=${ldapUser.username} email=${ldapUser.email} displayName=${ldapUser.displayName}`);
log(`syncUsers: [updating user] username=${ldapUser.username} email=${ldapUser.email} displayName=${ldapUser.displayName}`);
const [userUpdateError] = await safe(users.update(user, { email: ldapUser.email, fallbackEmail: ldapUser.email, displayName: ldapUser.displayName }, AuditSource.EXTERNAL_LDAP));
if (userUpdateError) debug('Failed to update user. %j %o', ldapUser, userUpdateError);
if (userUpdateError) log('Failed to update user. %j %o', ldapUser, userUpdateError);
} else {
// user known and up-to-date
debug(`syncUsers: [up-to-date user] username=${ldapUser.username} email=${ldapUser.email} displayName=${ldapUser.displayName}`);
log(`syncUsers: [up-to-date user] username=${ldapUser.username} email=${ldapUser.email} displayName=${ldapUser.displayName}`);
}
}
debug('syncUsers: done');
log('syncUsers: done');
}
async function syncGroups(config, progressCallback) {
@@ -397,15 +397,15 @@ async function syncGroups(config, progressCallback) {
assert.strictEqual(typeof progressCallback, 'function');
if (!config.syncGroups) {
debug('syncGroups: Group sync is disabled');
log('syncGroups: Group sync is disabled');
progressCallback({ percent: 70, message: 'Skipping group sync' });
return [];
}
const ldapGroups = await ldapGroupSearch(config, {});
debug(`syncGroups: Found ${ldapGroups.length} groups:`);
debug(ldapGroups);
log(`syncGroups: Found ${ldapGroups.length} groups:`);
log(ldapGroups);
let percent = 40;
const step = 28 / (ldapGroups.length + 1);
@@ -423,19 +423,19 @@ async function syncGroups(config, progressCallback) {
const result = await groups.getByName(groupName);
if (!result) {
debug(`syncGroups: [adding group] groupname=${groupName}`);
log(`syncGroups: [adding group] groupname=${groupName}`);
const [error] = await safe(groups.add({ name: groupName, source: 'ldap' }, AuditSource.EXTERNAL_LDAP));
if (error) debug('syncGroups: Failed to create group', groupName, error);
if (error) log('syncGroups: Failed to create group', groupName, error);
} else {
// convert local group to ldap group. 2 reasons:
// 1. we reset source flag when externalldap is disabled. if we renable, it automatically converts
// 2. externalldap connector usually implies user wants to user external users/groups.
groups.update(result.id, { source: 'ldap' });
debug(`syncGroups: [up-to-date group] groupname=${groupName}`);
log(`syncGroups: [up-to-date group] groupname=${groupName}`);
}
}
debug('syncGroups: sync done');
log('syncGroups: sync done');
}
async function syncGroupMembers(config, progressCallback) {
@@ -443,14 +443,14 @@ async function syncGroupMembers(config, progressCallback) {
assert.strictEqual(typeof progressCallback, 'function');
if (!config.syncGroups) {
debug('syncGroupMembers: Group users sync is disabled');
log('syncGroupMembers: Group users sync is disabled');
progressCallback({ percent: 98, message: 'Skipping group member sync' });
return [];
}
const allGroups = await groups.listWithMembers();
const ldapGroups = allGroups.filter(function (g) { return g.source === 'ldap'; });
debug(`syncGroupMembers: Found ${ldapGroups.length} groups to sync users`);
log(`syncGroupMembers: Found ${ldapGroups.length} groups to sync users`);
let percent = 70;
const step = 28 / (ldapGroups.length + 1);
@@ -458,11 +458,11 @@ async function syncGroupMembers(config, progressCallback) {
for (const ldapGroup of ldapGroups) {
percent = Math.min(percent + step, 98);
progressCallback({ percent: Math.round(percent), message: `Syncing members of ${ldapGroup.name}` });
debug(`syncGroupMembers: Sync users for group ${ldapGroup.name}`);
log(`syncGroupMembers: Sync users for group ${ldapGroup.name}`);
const result = await ldapGroupSearch(config, {});
if (!result || result.length === 0) {
debug(`syncGroupMembers: Unable to find group ${ldapGroup.name} ignoring for now.`);
log(`syncGroupMembers: Unable to find group ${ldapGroup.name} ignoring for now.`);
continue;
}
@@ -473,7 +473,7 @@ async function syncGroupMembers(config, progressCallback) {
});
if (!found) {
debug(`syncGroupMembers: Unable to find group ${ldapGroup.name} ignoring for now.`);
log(`syncGroupMembers: Unable to find group ${ldapGroup.name} ignoring for now.`);
continue;
}
@@ -482,24 +482,24 @@ async function syncGroupMembers(config, progressCallback) {
// if only one entry is in the group ldap returns a string, not an array!
if (typeof ldapGroupMembers === 'string') ldapGroupMembers = [ ldapGroupMembers ];
debug(`syncGroupMembers: Group ${ldapGroup.name} has ${ldapGroupMembers.length} members.`);
log(`syncGroupMembers: Group ${ldapGroup.name} has ${ldapGroupMembers.length} members.`);
const userIds = [];
for (const memberDn of ldapGroupMembers) {
const [ldapError, memberResult] = await safe(ldapGetByDN(config, memberDn));
if (ldapError) {
debug(`syncGroupMembers: Group ${ldapGroup.name} failed to get ${memberDn}: %o`, ldapError);
log(`syncGroupMembers: Group ${ldapGroup.name} failed to get ${memberDn}: %o`, ldapError);
continue;
}
debug(`syncGroupMembers: Group ${ldapGroup.name} has member object ${memberDn}`);
log(`syncGroupMembers: Group ${ldapGroup.name} has member object ${memberDn}`);
const username = memberResult[config.usernameField]?.toLowerCase();
if (!username) continue;
const [getError, userObject] = await safe(users.getByUsername(username));
if (getError || !userObject) {
debug(`syncGroupMembers: Failed to get user by username ${username}. %o`, getError ? getError : 'User not found');
log(`syncGroupMembers: Failed to get user by username ${username}. %o`, getError ? getError : 'User not found');
continue;
}
@@ -507,15 +507,15 @@ async function syncGroupMembers(config, progressCallback) {
}
const membersChanged = ldapGroup.userIds.length !== userIds.length || ldapGroup.userIds.some(id => !userIds.includes(id));
if (membersChanged) {
debug(`syncGroupMembers: Group ${ldapGroup.name} changed.`);
log(`syncGroupMembers: Group ${ldapGroup.name} changed.`);
const [setError] = await safe(groups.setMembers(ldapGroup, userIds, { skipSourceCheck: true }, AuditSource.EXTERNAL_LDAP));
if (setError) debug(`syncGroupMembers: Failed to set members of group ${ldapGroup.name}. %o`, setError);
if (setError) log(`syncGroupMembers: Failed to set members of group ${ldapGroup.name}. %o`, setError);
} else {
debug(`syncGroupMembers: Group ${ldapGroup.name} is unchanged.`);
log(`syncGroupMembers: Group ${ldapGroup.name} is unchanged.`);
}
}
debug('syncGroupMembers: done');
log('syncGroupMembers: done');
}
async function sync(progressCallback) {
@@ -532,7 +532,7 @@ async function sync(progressCallback) {
progressCallback({ percent: 100, message: 'Done' });
debug('sync: done');
log('sync: done');
}
export default {

6
src/hush.js
View File

@@ -1,10 +1,10 @@
import assert from 'node:assert';
import BoxError from './boxerror.js';
import crypto from 'node:crypto';
import debugModule from 'debug';
import logger from './logger.js';
import { Transform as TransformStream } from 'node:stream';
const debug = debugModule('box:hush');
const { log, trace } = logger('hush');
class EncryptStream extends TransformStream {
constructor(encryption) {
@@ -144,7 +144,7 @@ function decryptFilePath(filePath, encryption) {
decryptedParts.push(plainTextString);
} catch (error) {
debug(`Error decrypting part ${part} of path ${filePath}: %o`, error);
log(`Error decrypting part ${part} of path ${filePath}: %o`, error);
return { error: new BoxError(BoxError.CRYPTO_ERROR, `Decryption error. ${part} of path ${filePath}: ${error.message}`) };
}
}

18
src/janitor.js
View File

@@ -1,22 +1,22 @@
import assert from 'node:assert';
import BoxError from './boxerror.js';
import debugModule from 'debug';
import logger from './logger.js';
import Docker from 'dockerode';
import safe from 'safetydance';
import tokens from './tokens.js';
const debug = debugModule('box:janitor');
const { log, trace } = logger('janitor');
const gConnection = new Docker({ socketPath: '/var/run/docker.sock' });
async function cleanupTokens() {
debug('Cleaning up expired tokens');
log('Cleaning up expired tokens');
const [error, result] = await safe(tokens.delExpired());
if (error) return debug('cleanupTokens: error removing expired tokens. %o', error);
if (error) return log('cleanupTokens: error removing expired tokens. %o', error);
debug(`Cleaned up ${result} expired tokens`);
log(`Cleaned up ${result} expired tokens`);
}
async function cleanupTmpVolume(containerInfo) {
@@ -24,7 +24,7 @@ async function cleanupTmpVolume(containerInfo) {
const cmd = 'find /tmp -type f -mtime +10 -exec rm -rf {} +'.split(' '); // 10 day old files
debug(`cleanupTmpVolume ${JSON.stringify(containerInfo.Names)}`);
log(`cleanupTmpVolume ${JSON.stringify(containerInfo.Names)}`);
const [error, execContainer] = await safe(gConnection.getContainer(containerInfo.Id).exec({ Cmd: cmd, AttachStdout: true, AttachStderr: true, Tty: false }));
if (error) throw new BoxError(BoxError.DOCKER_ERROR, `Failed to exec container: ${error.message}`);
@@ -41,16 +41,16 @@ async function cleanupTmpVolume(containerInfo) {
}
async function cleanupDockerVolumes() {
debug('Cleaning up docker volumes');
log('Cleaning up docker volumes');
const [error, containers] = await safe(gConnection.listContainers({ all: 0 }));
if (error) throw new BoxError(BoxError.DOCKER_ERROR, error);
for (const container of containers) {
await safe(cleanupTmpVolume(container), { debug }); // intentionally ignore error
await safe(cleanupTmpVolume(container), { debug: log }); // intentionally ignore error
}
debug('Cleaned up docker volumes');
log('Cleaned up docker volumes');
}
export default {

46
src/ldapserver.js
View File

@@ -4,7 +4,7 @@ import apps from './apps.js';
import AuditSource from './auditsource.js';
import BoxError from './boxerror.js';
import constants from './constants.js';
import debugModule from 'debug';
import logger from './logger.js';
import eventlog from './eventlog.js';
import groups from './groups.js';
import ldap from 'ldapjs';
@@ -13,7 +13,7 @@ import safe from 'safetydance';
import users from './users.js';
import util from 'node:util';
const debug = debugModule('box:ldapserver');
const { log, trace } = logger('ldapserver');
let _MOCK_APP = null;
@@ -142,7 +142,7 @@ function finalSend(results, req, res, next) {
}
async function userSearch(req, res, next) {
debug('user search: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id);
log('user search: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id);
const [error, result] = await safe(getUsersWithAccessToApp(req));
if (error) return next(new ldap.OperationsError(error.message));
@@ -195,7 +195,7 @@ async function userSearch(req, res, next) {
}
async function groupSearch(req, res, next) {
debug('group search: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id);
log('group search: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id);
const results = [];
@@ -230,7 +230,7 @@ async function groupSearch(req, res, next) {
}
async function groupUsersCompare(req, res, next) {
debug('group users compare: dn %s, attribute %s, value %s (from %s)', req.dn.toString(), req.attribute, req.value, req.connection.ldap.id);
log('group users compare: dn %s, attribute %s, value %s (from %s)', req.dn.toString(), req.attribute, req.value, req.connection.ldap.id);
const [error, result] = await safe(getUsersWithAccessToApp(req));
if (error) return next(new ldap.OperationsError(error.message));
@@ -245,7 +245,7 @@ async function groupUsersCompare(req, res, next) {
}
async function groupAdminsCompare(req, res, next) {
debug('group admins compare: dn %s, attribute %s, value %s (from %s)', req.dn.toString(), req.attribute, req.value, req.connection.ldap.id);
log('group admins compare: dn %s, attribute %s, value %s (from %s)', req.dn.toString(), req.attribute, req.value, req.connection.ldap.id);
const [error, result] = await safe(getUsersWithAccessToApp(req));
if (error) return next(new ldap.OperationsError(error.message));
@@ -260,7 +260,7 @@ async function groupAdminsCompare(req, res, next) {
}
async function mailboxSearch(req, res, next) {
debug('mailbox search: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id);
log('mailbox search: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id);
// if cn is set OR filter is mail= we only search for one mailbox specifically
let email, dn;
@@ -350,7 +350,7 @@ async function mailboxSearch(req, res, next) {
}
async function mailAliasSearch(req, res, next) {
debug('mail alias get: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id);
log('mail alias get: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id);
if (!req.dn.rdns[0].attrs.cn) return next(new ldap.NoSuchObjectError('Missing CN'));
@@ -389,7 +389,7 @@ async function mailAliasSearch(req, res, next) {
}
async function mailingListSearch(req, res, next) {
debug('mailing list get: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id);
log('mailing list get: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id);
if (!req.dn.rdns[0].attrs.cn) return next(new ldap.NoSuchObjectError('Missing CN'));
@@ -433,7 +433,7 @@ async function mailingListSearch(req, res, next) {
// Will attach req.user if successful
async function authenticateUser(req, res, next) {
debug('user bind: %s (from %s)', req.dn.toString(), req.connection.ldap.id);
log('user bind: %s (from %s)', req.dn.toString(), req.connection.ldap.id);
const appId = req.app.id;
@@ -478,7 +478,7 @@ async function verifyMailboxPassword(mailbox, password) {
}
async function authenticateSftp(req, res, next) {
debug('sftp auth: %s (from %s)', req.dn.toString(), req.connection.ldap.id);
log('sftp auth: %s (from %s)', req.dn.toString(), req.connection.ldap.id);
if (!req.dn.rdns[0].attrs.cn) return next(new ldap.NoSuchObjectError('Missing CN'));
@@ -492,13 +492,13 @@ async function authenticateSftp(req, res, next) {
const [verifyError] = await safe(users.verifyWithUsername(parts[0], req.credentials, app.id, { skipTotpCheck: true }));
if (verifyError) return next(new ldap.InvalidCredentialsError(verifyError.message));
debug('sftp auth: success');
log('sftp auth: success');
res.end();
}
async function userSearchSftp(req, res, next) {
debug('sftp user search: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id);
log('sftp user search: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id);
if (req.filter.attribute !== 'username' || !req.filter.value) return next(new ldap.NoSuchObjectError());
@@ -556,7 +556,7 @@ async function verifyAppMailboxPassword(serviceId, username, password) {
}
async function authenticateService(serviceId, dn, req, res, next) {
debug(`authenticateService: ${req.dn.toString()} (from ${req.connection.ldap.id})`);
log(`authenticateService: ${req.dn.toString()} (from ${req.connection.ldap.id})`);
if (!dn.rdns[0].attrs.cn) return next(new ldap.NoSuchObjectError(dn.toString()));
@@ -608,7 +608,7 @@ async function authenticateMail(req, res, next) {
// https://ldapwiki.com/wiki/RootDSE / RFC 4512 - ldapsearch -x -h "${CLOUDRON_LDAP_SERVER}" -p "${CLOUDRON_LDAP_PORT}" -b "" -s base
// ldapjs seems to call this handler for everything when search === ''
async function maybeRootDSE(req, res, next) {
debug(`maybeRootDSE: requested with scope:${req.scope} dn:${req.dn.toString()}`);
log(`maybeRootDSE: requested with scope:${req.scope} dn:${req.dn.toString()}`);
if (req.scope !== 'base') return next(new ldap.NoSuchObjectError()); // per the spec, rootDSE search require base scope
if (!req.dn || req.dn.toString() !== '') return next(new ldap.NoSuchObjectError());
@@ -633,16 +633,16 @@ async function start() {
const logger = {
trace: NOOP,
debug: NOOP,
info: debug,
warn: debug,
error: debug,
fatal: debug
info: log,
warn: log,
error: log,
fatal: log
};
gServer = ldap.createServer({ log: logger });
gServer.on('error', function (error) {
debug('start: server error. %o', error);
log('start: server error. %o', error);
});
gServer.search('ou=users,dc=cloudron', authenticateApp, userSearch);
@@ -670,14 +670,14 @@ async function start() {
// this is the bind for addons (after bind, they might search and authenticate)
gServer.bind('ou=addons,dc=cloudron', function(req, res /*, next */) {
debug('addons bind: %s', req.dn.toString()); // note: cn can be email or id
log('addons bind: %s', req.dn.toString()); // note: cn can be email or id
res.end();
});
// this is the bind for apps (after bind, they might search and authenticate user)
gServer.bind('ou=apps,dc=cloudron', function(req, res /*, next */) {
// TODO: validate password
debug('application bind: %s', req.dn.toString());
log('application bind: %s', req.dn.toString());
res.end();
});
@@ -693,7 +693,7 @@ async function start() {
// just log that an attempt was made to unknown route, this helps a lot during app packaging
gServer.use(function(req, res, next) {
debug('not handled: dn %s, scope %s, filter %s (from %s)', req.dn ? req.dn.toString() : '-', req.scope, req.filter ? req.filter.toString() : '-', req.connection.ldap.id);
log('not handled: dn %s, scope %s, filter %s (from %s)', req.dn ? req.dn.toString() : '-', req.scope, req.filter ? req.filter.toString() : '-', req.connection.ldap.id);
return next();
});

24
src/locks.js
View File

@@ -1,10 +1,10 @@
import assert from 'node:assert';
import BoxError from './boxerror.js';
import database from './database.js';
import debugModule from 'debug';
import logger from './logger.js';
import promiseRetry from './promise-retry.js';
const debug = debugModule('box:locks');
const { log, trace } = logger('locks');
const TYPE_APP_TASK_PREFIX = 'app_task_';
const TYPE_APP_BACKUP_PREFIX = 'app_backup_';
@@ -31,7 +31,7 @@ async function write(value) {
const result = await database.query('UPDATE locks SET dataJson=?, version=version+1 WHERE id=? AND version=?', [ JSON.stringify(value.data), 'platform', value.version ]);
if (result.affectedRows !== 1) throw new BoxError(BoxError.CONFLICT, 'Someone updated before we did');
debug(`write: current locks: ${JSON.stringify(value.data)}`);
log(`write: current locks: ${JSON.stringify(value.data)}`);
}
function canAcquire(data, type) {
@@ -59,58 +59,58 @@ function canAcquire(data, type) {
async function acquire(type) {
assert.strictEqual(typeof type, 'string');
await promiseRetry({ times: Number.MAX_SAFE_INTEGER, interval: 100, debug, retry: (error) => error.reason === BoxError.CONFLICT }, async () => {
await promiseRetry({ times: Number.MAX_SAFE_INTEGER, interval: 100, debug: log, retry: (error) => error.reason === BoxError.CONFLICT }, async () => {
const { version, data } = await read();
const error = canAcquire(data, type);
if (error) throw error;
data[type] = gTaskId;
await write({ version, data });
debug(`acquire: ${type}`);
log(`acquire: ${type}`);
});
}
async function wait(type) {
assert.strictEqual(typeof type, 'string');
await promiseRetry({ times: Number.MAX_SAFE_INTEGER, interval: 10000, debug }, async () => await acquire(type));
await promiseRetry({ times: Number.MAX_SAFE_INTEGER, interval: 10000, debug: log }, async () => await acquire(type));
}
async function release(type) {
assert.strictEqual(typeof type, 'string');
await promiseRetry({ times: Number.MAX_SAFE_INTEGER, interval: 100, debug, retry: (error) => error.reason === BoxError.CONFLICT }, async () => {
await promiseRetry({ times: Number.MAX_SAFE_INTEGER, interval: 100, debug: log, retry: (error) => error.reason === BoxError.CONFLICT }, async () => {
const { version, data } = await read();
if (!(type in data)) throw new BoxError(BoxError.BAD_STATE, `Lock ${type} was never acquired`);
if (data[type] !== gTaskId) throw new BoxError(BoxError.BAD_STATE, `Task ${gTaskId} attempted to release lock ${type} acquired by ${data[type]}`);
delete data[type];
await write({ version, data });
debug(`release: ${type}`);
log(`release: ${type}`);
});
}
async function releaseAll() {
await database.query('DELETE FROM locks');
await database.query('INSERT INTO locks (id, dataJson) VALUES (?, ?)', [ 'platform', JSON.stringify({}) ]);
debug('releaseAll: all locks released');
log('releaseAll: all locks released');
}
// identify programming errors in tasks that forgot to clean up locks
async function releaseByTaskId(taskId) {
assert.strictEqual(typeof taskId, 'string');
await promiseRetry({ times: Number.MAX_SAFE_INTEGER, interval: 100, debug, retry: (error) => error.reason === BoxError.CONFLICT }, async () => {
await promiseRetry({ times: Number.MAX_SAFE_INTEGER, interval: 100, debug: log, retry: (error) => error.reason === BoxError.CONFLICT }, async () => {
const { version, data } = await read();
for (const type of Object.keys(data)) {
if (data[type] === taskId) {
debug(`releaseByTaskId: task ${taskId} forgot to unlock ${type}`);
log(`releaseByTaskId: task ${taskId} forgot to unlock ${type}`);
delete data[type];
}
}
await write({ version, data });
debug(`releaseByTaskId: ${taskId}`);
log(`releaseByTaskId: ${taskId}`);
});
}

16
src/logger.js Normal file
View File

@@ -0,0 +1,16 @@
import util from 'node:util';
const TRACE_ENABLED = false;
const LOG_ENABLED = process.env.BOX_ENV !== 'test' || !!process.env.LOG;
function output(namespace, args) {
const ts = new Date().toISOString();
process.stdout.write(`${ts} ${namespace}: ${util.format(...args)}\n`);
}
export default function logger(namespace) {
return {
log: LOG_ENABLED ? (...args) => output(namespace, args) : () => {},
trace: TRACE_ENABLED ? (...args) => output(namespace, args) : () => {},
};
}

6
src/logs.js
View File

@@ -1,11 +1,11 @@
import assert from 'node:assert';
import child_process from 'node:child_process';
import debugModule from 'debug';
import logger from './logger.js';
import path from 'node:path';
import stream from 'node:stream';
import { StringDecoder } from 'node:string_decoder';
const debug = debugModule('box:logs');
const { log, trace } = logger('logs');
const TransformStream = stream.Transform;
const LOGTAIL_CMD = path.join(import.meta.dirname, 'scripts/logtail.sh');
@@ -74,7 +74,7 @@ function tail(filePaths, options) {
cp.terminate = () => {
child_process.execFile('/usr/bin/sudo', [ KILL_CHILD_CMD, cp.pid, process.pid ], { encoding: 'utf8' }, (error, stdout, stderr) => {
if (error) debug(`tail: failed to kill children`, stdout, stderr);
if (error) log(`tail: failed to kill children`, stdout, stderr);
});
};

36
src/mail.js
View File

@@ -2,7 +2,7 @@ import assert from 'node:assert';
import BoxError from './boxerror.js';
import constants from './constants.js';
import database from './database.js';
import debugModule from 'debug';
import logger from './logger.js';
import dig from './dig.js';
import dns from './dns.js';
import eventlog from './eventlog.js';
@@ -22,7 +22,7 @@ import superagent from '@cloudron/superagent';
import validator from './validator.js';
import _ from './underscore.js';
const debug = debugModule('box:mail');
const { log, trace } = logger('mail');
const shell = shellModule('mail');
const OWNERTYPE_USER = 'user';
@@ -524,14 +524,14 @@ async function checkRbl(type, mailDomain) {
const [rblError, records] = await safe(dig.resolve(`${flippedIp}.${rblServer.dns}`, 'A', DNS_OPTIONS));
if (rblError || records.length === 0) continue; // not listed
debug(`checkRbl (${domain}) flippedIp: ${flippedIp} is in the blocklist of ${rblServer.dns}: ${JSON.stringify(records)}`);
log(`checkRbl (${domain}) flippedIp: ${flippedIp} is in the blocklist of ${rblServer.dns}: ${JSON.stringify(records)}`);
const result = Object.assign({}, rblServer);
const [error2, txtRecords] = await safe(dig.resolve(`${flippedIp}.${rblServer.dns}`, 'TXT', DNS_OPTIONS));
result.txtRecords = error2 || !txtRecords ? [] : txtRecords.map(x => x.join(''));
debug(`checkRbl (${domain}) error: ${error2?.message || null} txtRecords: ${JSON.stringify(txtRecords)}`);
log(`checkRbl (${domain}) error: ${error2?.message || null} txtRecords: ${JSON.stringify(txtRecords)}`);
blockedServers.push(result);
}
@@ -573,11 +573,11 @@ async function getStatus(domain) {
for (let i = 0; i < checks.length; i++) {
const response = responses[i], check = checks[i];
if (response.status !== 'fulfilled') {
debug(`check ${check.what} was rejected. This is not expected. reason: ${response.reason}`);
log(`check ${check.what} was rejected. This is not expected. reason: ${response.reason}`);
continue;
}
if (response.value.message) debug(`${check.what} (${domain}): ${response.value.message}`);
if (response.value.message) log(`${check.what} (${domain}): ${response.value.message}`);
safe.set(results, checks[i].what, response.value || {});
}
@@ -627,7 +627,7 @@ async function txtRecordsWithSpf(domain, mailFqdn) {
const txtRecords = await dns.getDnsRecords('', domain, 'TXT');
debug('txtRecordsWithSpf: current txt records - %j', txtRecords);
log('txtRecordsWithSpf: current txt records - %j', txtRecords);
let i, matches, validSpf;
@@ -644,10 +644,10 @@ async function txtRecordsWithSpf(domain, mailFqdn) {
if (!matches) { // no spf record was found, create one
txtRecords.push('"v=spf1 a:' + mailFqdn + ' ~all"');
debug('txtRecordsWithSpf: adding txt record');
log('txtRecordsWithSpf: adding txt record');
} else { // just add ourself
txtRecords[i] = matches[1] + ' a:' + mailFqdn + txtRecords[i].slice(matches[1].length);
debug('txtRecordsWithSpf: inserting txt record');
log('txtRecordsWithSpf: inserting txt record');
}
return txtRecords;
@@ -657,7 +657,7 @@ async function upsertDnsRecords(domain, mailFqdn) {
assert.strictEqual(typeof domain, 'string');
assert.strictEqual(typeof mailFqdn, 'string');
debug(`upsertDnsRecords: updating mail dns records domain:${domain} mailFqdn:${mailFqdn}`);
log(`upsertDnsRecords: updating mail dns records domain:${domain} mailFqdn:${mailFqdn}`);
const mailDomain = await getDomain(domain);
if (!mailDomain) throw new BoxError(BoxError.NOT_FOUND, 'mail domain not found');
@@ -679,13 +679,13 @@ async function upsertDnsRecords(domain, mailFqdn) {
const dmarcRecords = await dns.getDnsRecords('_dmarc', domain, 'TXT'); // only update dmarc if absent. this allows user to set email for reporting
if (dmarcRecords.length === 0) records.push({ subdomain: '_dmarc', domain, type: 'TXT', values: [ '"v=DMARC1; p=reject; pct=100"' ] });
debug(`upsertDnsRecords: updating ${domain} with ${records.length} records: ${JSON.stringify(records)}`);
log(`upsertDnsRecords: updating ${domain} with ${records.length} records: ${JSON.stringify(records)}`);
for (const record of records) {
await dns.upsertDnsRecords(record.subdomain, record.domain, record.type, record.values);
}
debug(`upsertDnsRecords: records of ${domain} added`);
log(`upsertDnsRecords: records of ${domain} added`);
}
async function setDnsRecords(domain) {
@@ -714,7 +714,7 @@ async function setMailFromValidation(domain, enabled) {
await updateDomain(domain, { mailFromValidation: enabled });
safe(mailServer.restart(), { debug }); // have to restart mail container since haraka cannot watch symlinked config files (mail.ini)
safe(mailServer.restart(), { debug: log }); // have to restart mail container since haraka cannot watch symlinked config files (mail.ini)
}
async function setBanner(domain, banner) {
@@ -723,7 +723,7 @@ async function setBanner(domain, banner) {
await updateDomain(domain, { banner });
safe(mailServer.restart(), { debug });
safe(mailServer.restart(), { debug: log });
}
async function setCatchAllAddress(domain, addresses) {
@@ -736,7 +736,7 @@ async function setCatchAllAddress(domain, addresses) {
await updateDomain(domain, { catchAll: addresses });
safe(mailServer.restart(), { debug }); // have to restart mail container since haraka cannot watch symlinked config files (mail.ini)
safe(mailServer.restart(), { debug: log }); // have to restart mail container since haraka cannot watch symlinked config files (mail.ini)
}
async function setMailRelay(domain, relay, options) {
@@ -760,7 +760,7 @@ async function setMailRelay(domain, relay, options) {
await updateDomain(domain, { relay });
safe(mailServer.restart(), { debug });
safe(mailServer.restart(), { debug: log });
}
async function setMailEnabled(domain, enabled, auditSource) {
@@ -977,7 +977,7 @@ async function delMailbox(name, domain, options, auditSource) {
if (result.affectedRows === 0) throw new BoxError(BoxError.NOT_FOUND, 'Mailbox not found');
const [error] = await safe(removeSolrIndex(mailbox));
if (error) debug(`delMailbox: failed to remove solr index: ${error.message}`);
if (error) log(`delMailbox: failed to remove solr index: ${error.message}`);
await eventlog.add(eventlog.ACTION_MAIL_MAILBOX_REMOVE, auditSource, { name, domain });
}
@@ -1190,7 +1190,7 @@ async function resolveMailingList(listName, listDomain) {
const member =`${memberName}@${memberDomain}`; // cleaned up without any '+' subaddress
if (visited.includes(member)) {
debug(`resolveMailingList: list ${listName}@${listDomain} has a recursion at member ${member}`);
log(`resolveMailingList: list ${listName}@${listDomain} has a recursion at member ${member}`);
continue;
}
visited.push(member);

10
src/mailer.js
View File

@@ -3,7 +3,7 @@ import BoxError from './boxerror.js';
import branding from './branding.js';
import constants from './constants.js';
import dashboard from './dashboard.js';
import debugModule from 'debug';
import logger from './logger.js';
import ejs from 'ejs';
import mailServer from './mailserver.js';
import nodemailer from 'nodemailer';
@@ -11,7 +11,7 @@ import path from 'node:path';
import safe from 'safetydance';
import translations from './translations.js';
const debug = debugModule('box:mailer');
const { log, trace } = logger('mailer');
const _mailQueue = []; // accumulate mails in test mode;
@@ -63,7 +63,7 @@ async function sendMail(mailOptions) {
const [error] = await safe(transport.sendMail(mailOptions));
if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, `Error sending Email "${mailOptions.subject}" to ${mailOptions.to}: ${error.message}`);
debug(`Email "${mailOptions.subject}" sent to ${mailOptions.to}`);
log(`Email "${mailOptions.subject}" sent to ${mailOptions.to}`);
}
function render(templateFile, params, translationAssets) {
@@ -73,7 +73,7 @@ function render(templateFile, params, translationAssets) {
let content = null;
let raw = safe.fs.readFileSync(path.join(MAIL_TEMPLATES_DIR, templateFile), 'utf8');
if (raw === null) {
debug(`Error loading ${templateFile}`);
log(`Error loading ${templateFile}`);
return '';
}
@@ -82,7 +82,7 @@ function render(templateFile, params, translationAssets) {
try {
content = ejs.render(raw, params);
} catch (e) {
debug(`Error rendering ${templateFile}`, e);
log(`Error rendering ${templateFile}`, e);
}
return content;

28
src/mailserver.js
View File

@@ -1,7 +1,7 @@
import assert from 'node:assert';
import BoxError from './boxerror.js';
import constants from './constants.js';
import debugModule from 'debug';
import logger from './logger.js';
import dns from './dns.js';
import docker from './docker.js';
import domains from './domains.js';
@@ -22,7 +22,7 @@ import shellModule from './shell.js';
import tasks from './tasks.js';
import users from './users.js';
const debug = debugModule('box:mailserver');
const { log, trace } = logger('mailserver');
const shell = shellModule('mailserver');
const DEFAULT_MEMORY_LIMIT = 512 * 1024 * 1024;
@@ -31,7 +31,7 @@ const DEFAULT_MEMORY_LIMIT = 512 * 1024 * 1024;
async function createMailConfig(mailFqdn) {
assert.strictEqual(typeof mailFqdn, 'string');
debug(`createMailConfig: generating mail config with ${mailFqdn}`);
log(`createMailConfig: generating mail config with ${mailFqdn}`);
const mailDomains = await mail.listDomains();
@@ -124,7 +124,7 @@ async function configureMail(mailFqdn, mailDomain, serviceConfig) {
// if the 'yellowtent' user of OS and the 'cloudron' user of mail container don't match, the keys become inaccessible by mail code
if (!safe.fs.chmodSync(mailKeyFilePath, 0o644)) throw new BoxError(BoxError.FS_ERROR, `Could not chmod key file: ${safe.error.message}`);
debug('configureMail: stopping and deleting previous mail container');
log('configureMail: stopping and deleting previous mail container');
await docker.stopContainer('mail');
await docker.deleteContainer('mail');
@@ -156,7 +156,7 @@ async function configureMail(mailFqdn, mailDomain, serviceConfig) {
--label isCloudronManaged=true \
${readOnly} -v /run -v /tmp ${image} ${cmd}`;
debug('configureMail: starting mail container');
log('configureMail: starting mail container');
await shell.bash(runCmd, { encoding: 'utf8' });
}
@@ -172,7 +172,7 @@ async function restart() {
const mailConfig = await services.getServiceConfig('mail');
const { domain, fqdn } = await getLocation();
debug(`restart: restarting mail container with mailFqdn:${fqdn} mailDomain:${domain}`);
log(`restart: restarting mail container with mailFqdn:${fqdn} mailDomain:${domain}`);
// NOTE: the email container has to be re-created. this is because some of the settings like solr config rely on starting with a clean /run state
await locks.wait(locks.TYPE_MAIL_SERVER_RESTART);
@@ -184,7 +184,7 @@ async function restart() {
async function start(existingInfra) {
assert.strictEqual(typeof existingInfra, 'object');
debug('startMail: starting');
log('startMail: starting');
await restart();
if (existingInfra.version !== 'none' && existingInfra.images.mail !== infra.images.mail) await docker.deleteImage(existingInfra.images.mail);
@@ -194,11 +194,11 @@ async function restartIfActivated() {
const activated = await users.isActivated();
if (!activated) {
debug('restartIfActivated: skipping restart of mail container since Cloudron is not activated yet');
log('restartIfActivated: skipping restart of mail container since Cloudron is not activated yet');
return; // not provisioned yet, do not restart container after dns setup
}
debug('restartIfActivated: restarting on activated');
log('restartIfActivated: restarting on activated');
await restart();
}
@@ -208,7 +208,7 @@ async function onDomainAdded(domain) {
const { fqdn } = await getLocation();
if (!fqdn) return; // mail domain is not set yet (when provisioning)
debug(`onDomainAdded: configuring mail for added domain ${domain}`);
log(`onDomainAdded: configuring mail for added domain ${domain}`);
await mail.upsertDnsRecords(domain, fqdn);
await restartIfActivated();
}
@@ -216,7 +216,7 @@ async function onDomainAdded(domain) {
async function onDomainRemoved(domain) {
assert.strictEqual(typeof domain, 'string');
debug(`onDomainRemoved: configuring mail for removed domain ${domain}`);
log(`onDomainRemoved: configuring mail for removed domain ${domain}`);
await restart();
}
@@ -224,10 +224,10 @@ async function checkCertificate() {
const certificate = await reverseProxy.getMailCertificate();
const cert = safe.fs.readFileSync(`${paths.MAIL_CONFIG_DIR}/tls_cert.pem`, { encoding: 'utf8' });
if (cert === certificate.cert) {
debug('checkCertificate: certificate has not changed');
log('checkCertificate: certificate has not changed');
return;
}
debug('checkCertificate: certificate has changed');
log('checkCertificate: certificate has changed');
await restartIfActivated();
}
@@ -289,7 +289,7 @@ async function startChangeLocation(subdomain, domain, auditSource) {
.then(async () => {
await platform.onMailServerLocationChanged(auditSource);
})
.catch((taskError) => debug(`startChangeLocation`, taskError));
.catch((taskError) => log(`startChangeLocation`, taskError));
await eventlog.add(eventlog.ACTION_MAIL_LOCATION, auditSource, { subdomain, domain, taskId });
return taskId;

12
src/metrics.js
View File

@@ -2,7 +2,7 @@ import apps from './apps.js';
import assert from 'node:assert';
import BoxError from './boxerror.js';
import constants from './constants.js';
import debugModule from 'debug';
import logger from './logger.js';
import docker from './docker.js';
import fs from 'node:fs';
import net from 'node:net';
@@ -15,7 +15,7 @@ import shellModule from './shell.js';
import superagent from '@cloudron/superagent';
import _ from './underscore.js';
const debug = debugModule('box:metrics');
const { log, trace } = logger('metrics');
const shell = shellModule('metrics');
@@ -163,7 +163,7 @@ async function readSystemMetrics() {
}
async function sendToGraphite() {
// debug('sendStatsToGraphite: collecting stats');
// log('sendStatsToGraphite: collecting stats');
const result = await readSystemMetrics();
@@ -203,7 +203,7 @@ async function sendToGraphite() {
});
client.on('error', (error) => {
debug(`Error sending data to graphite: ${error.message}`);
log(`Error sending data to graphite: ${error.message}`);
resolve();
});
@@ -381,7 +381,7 @@ async function pipeContainerToMap(name, statsMap) {
// we used to poll before instead of a stream. but docker caches metrics internally and rate logic has to handle dups
const statsStream = await docker.getStats(name, { stream: true });
statsStream.on('error', (error) => debug(error));
statsStream.on('error', (error) => log(error));
statsStream.on('data', (data) => {
const stats = JSON.parse(data.toString('utf8'));
const metrics = translateContainerStatsSync(stats);
@@ -478,7 +478,7 @@ async function getStream(options) {
const INTERVAL_MSECS = 1000;
intervalId = setInterval(async () => {
if (options.system) await safe(pipeSystemToMap(statsMap), { debug });
if (options.system) await safe(pipeSystemToMap(statsMap), { debug: log });
const result = {};
const nowSecs = Date.now() / 1000; // to match graphite return value

6
src/mounts.js
View File

@@ -1,7 +1,7 @@
import assert from 'node:assert';
import BoxError from './boxerror.js';
import constants from './constants.js';
import debugModule from 'debug';
import logger from './logger.js';
import ejs from 'ejs';
import fs from 'node:fs';
import path from 'node:path';
@@ -9,7 +9,7 @@ import paths from './paths.js';
import safe from 'safetydance';
import shellModule from './shell.js';
const debug = debugModule('box:mounts');
const { log, trace } = logger('mounts');
const shell = shellModule('mounts');
const MOUNT_TYPE_FILESYSTEM = 'filesystem';
@@ -163,7 +163,7 @@ async function removeMount(mount) {
if (constants.TEST) return;
await safe(shell.sudo([ RM_MOUNT_CMD, hostPath ], {}), { debug }); // ignore any error
await safe(shell.sudo([ RM_MOUNT_CMD, hostPath ], {}), { debug: log }); // ignore any error
if (mountType === MOUNT_TYPE_SSHFS) {
const keyFilePath = path.join(paths.SSHFS_KEYS_DIR, `identity_file_${path.basename(hostPath)}`);

10
src/network/generic.js
View File

@@ -1,11 +1,11 @@
import assert from 'node:assert';
import BoxError from '../boxerror.js';
import constants from '../constants.js';
import debugModule from 'debug';
import logger from '../logger.js';
import safe from 'safetydance';
import superagent from '@cloudron/superagent';
const debug = debugModule('box:network/generic');
const { log, trace } = logger('network/generic');
const gCache = { ipv4: {}, ipv6: {} }; // each has { timestamp, value, request }
@@ -15,16 +15,16 @@ async function getIP(type) {
gCache[type].value = null; // clear the obsolete value
debug(`getIP: querying ${url} to get ${type}`);
log(`getIP: querying ${url} to get ${type}`);
const [networkError, response] = await safe(superagent.get(url).timeout(30 * 1000).retry(2).ok(() => true));
if (networkError || response.status !== 200) {
debug(`getIP: Error getting IP. ${networkError.message}`);
log(`getIP: Error getting IP. ${networkError.message}`);
throw new BoxError(BoxError.EXTERNAL_ERROR, `Unable to detect ${type}. API server (${type}.api.cloudron.io) unreachable`);
}
if (!response.body?.ip) {
debug('get: Unexpected answer. No "ip" found in response body.', response.body);
log('get: Unexpected answer. No "ip" found in response body.', response.body);
throw new BoxError(BoxError.EXTERNAL_ERROR, `Unable to detect ${type}. No IP found in response`);
}

8
src/network/network-interface.js
View File

@@ -1,10 +1,10 @@
import assert from 'node:assert';
import BoxError from '../boxerror.js';
import debugModule from 'debug';
import logger from '../logger.js';
import os from 'node:os';
import safe from 'safetydance';
const debug = debugModule('box:network/network-interface');
const { log, trace } = logger('network/network-interface');
async function getIPv4(config) {
@@ -16,7 +16,7 @@ async function getIPv4(config) {
const addresses = iface.filter(i => i.family === 'IPv4').map(i => i.address);
if (addresses.length === 0) throw new BoxError(BoxError.NETWORK_ERROR, `${config.ifname} does not have any IPv4 address`);
if (addresses.length > 1) debug(`${config.ifname} has multiple ipv4 - ${JSON.stringify(addresses)}. choosing the first one.`);
if (addresses.length > 1) log(`${config.ifname} has multiple ipv4 - ${JSON.stringify(addresses)}. choosing the first one.`);
return addresses[0];
}
@@ -30,7 +30,7 @@ async function getIPv6(config) {
const addresses = iface.filter(i => i.family === 'IPv6').map(i => i.address);
if (addresses.length === 0) throw new BoxError(BoxError.NETWORK_ERROR, `${config.ifname} does not have any IPv6 address`);
if (addresses.length > 1) debug(`${config.ifname} has multiple ipv6 - ${JSON.stringify(addresses)}. choosing the first one.`);
if (addresses.length > 1) log(`${config.ifname} has multiple ipv6 - ${JSON.stringify(addresses)}. choosing the first one.`);
return addresses[0];
}

22
src/notifications.js
View File

@@ -4,13 +4,13 @@ import BoxError from './boxerror.js';
import changelog from './changelog.js';
import dashboard from './dashboard.js';
import database from './database.js';
import debugModule from 'debug';
import logger from './logger.js';
import eventlog from './eventlog.js';
import mailer from './mailer.js';
import safe from 'safetydance';
import users from './users.js';
const debug = debugModule('box:notifications');
const { log, trace } = logger('notifications');
const TYPE_CLOUDRON_INSTALLED = 'cloudronInstalled';
const TYPE_CLOUDRON_UPDATED = 'cloudronUpdated';
@@ -44,7 +44,7 @@ async function add(type, title, message, data) {
assert.strictEqual(typeof message, 'string');
assert.strictEqual(typeof data, 'object');
debug(`add: ${type} ${title}`);
log(`add: ${type} ${title}`);
const query = 'INSERT INTO notifications (type, title, message, acknowledged, eventId, context) VALUES (?, ?, ?, ?, ?, ?)';
const args = [ type, title, message, false, data?.eventId || null, data.context || '' ];
@@ -147,7 +147,7 @@ async function oomEvent(eventId, containerId, app, addonName, event) {
const admins = await users.getAdmins();
for (const admin of admins) {
if (admin.notificationConfig.includes(TYPE_APP_OOM)) {
await safe(mailer.oomEvent(admin.email, containerId, app, addonName, event), { debug });
await safe(mailer.oomEvent(admin.email, containerId, app, addonName, event), { debug: log });
}
}
}
@@ -159,7 +159,7 @@ async function appUp(eventId, app) {
const admins = await users.getAdmins();
for (const admin of admins) {
if (admin.notificationConfig.includes(TYPE_APP_UP)) {
await safe(mailer.appUp(admin.email, app), { debug });
await safe(mailer.appUp(admin.email, app), { debug: log });
}
}
}
@@ -171,7 +171,7 @@ async function appDown(eventId, app) {
const admins = await users.getAdmins();
for (const admin of admins) {
if (admin.notificationConfig.includes(TYPE_APP_DOWN)) {
await safe(mailer.appDown(admin.email, app), { debug });
await safe(mailer.appDown(admin.email, app), { debug: log });
}
}
}
@@ -222,7 +222,7 @@ async function boxUpdateError(eventId, errorMessage) {
const admins = await users.getAdmins();
for (const admin of admins) {
if (admin.notificationConfig.includes(TYPE_CLOUDRON_UPDATE_FAILED)) {
await safe(mailer.boxUpdateError(admin.email, errorMessage), { debug });
await safe(mailer.boxUpdateError(admin.email, errorMessage), { debug: log });
}
}
}
@@ -237,7 +237,7 @@ async function certificateRenewalError(eventId, fqdn, errorMessage) {
const admins = await users.getAdmins();
for (const admin of admins) {
if (admin.notificationConfig.includes(TYPE_CERTIFICATE_RENEWAL_FAILED)) {
await safe(mailer.certificateRenewalError(admin.email, fqdn, errorMessage), { debug });
await safe(mailer.certificateRenewalError(admin.email, fqdn, errorMessage), { debug: log });
}
}
}
@@ -253,7 +253,7 @@ async function backupFailed(eventId, taskId, errorMessage) {
const superadmins = await users.getSuperadmins();
for (const superadmin of superadmins) {
if (superadmin.notificationConfig.includes(TYPE_BACKUP_FAILED)) {
await safe(mailer.backupFailed(superadmin.email, errorMessage, `https://${dashboardFqdn}/logs.html?taskId=${taskId}`), { debug });
await safe(mailer.backupFailed(superadmin.email, errorMessage, `https://${dashboardFqdn}/logs.html?taskId=${taskId}`), { debug: log });
}
}
}
@@ -262,7 +262,7 @@ async function rebootRequired() {
const admins = await users.getAdmins();
for (const admin of admins) {
if (admin.notificationConfig.includes(TYPE_REBOOT)) {
await safe(mailer.rebootRequired(admin.email), { debug });
await safe(mailer.rebootRequired(admin.email), { debug: log });
}
}
}
@@ -273,7 +273,7 @@ async function lowDiskSpace(message) {
const admins = await users.getAdmins();
for (const admin of admins) {
if (admin.notificationConfig.includes(TYPE_DISK_SPACE)) {
await safe(mailer.lowDiskSpace(admin.email, message), { debug });
await safe(mailer.lowDiskSpace(admin.email, message), { debug: log });
}
}
}

52
src/oidcserver.js
View File

@@ -7,7 +7,7 @@ import branding from './branding.js';
import constants from './constants.js';
import crypto from 'node:crypto';
import dashboard from './dashboard.js';
import debugModule from 'debug';
import logger from './logger.js';
import dns from './dns.js';
import ejs from 'ejs';
import express from 'express';
@@ -33,7 +33,7 @@ import util from 'node:util';
import Provider from 'oidc-provider';
import mailpasswords from './mailpasswords.js';
const debug = debugModule('box:oidcserver');
const { log, trace } = logger('oidcserver');
// 1. Index.vue starts the OIDC flow by navigating to /openid/auth. Webadmin sets callback url to authcallback.html + implicit flow
@@ -83,12 +83,12 @@ class StorageAdapter {
}
constructor(name) {
debug(`Creating OpenID storage adapter for ${name}`);
log(`Creating OpenID storage adapter for ${name}`);
this.name = name;
}
async upsert(id, payload, expiresIn) {
debug(`[${this.name}] upsert: ${id}`);
log(`[${this.name}] upsert: ${id}`);
const expiresAt = expiresIn ? new Date(Date.now() + (expiresIn * 1000)) : 0;
@@ -102,7 +102,7 @@ class StorageAdapter {
const [error] = await safe(tokens.add({ clientId: payload.clientId, identifier: user.id, expires, accessToken: id, allowedIpRanges: '' }));
if (error) {
debug('Error adding access token', error);
log('Error adding access token', error);
throw error;
}
} else {
@@ -111,12 +111,12 @@ class StorageAdapter {
}
async find(id) {
debug(`[${this.name}] find: ${id}`);
log(`[${this.name}] find: ${id}`);
if (this.name === 'Client') {
const [error, client] = await safe(oidcClients.get(id));
if (error || !client) {
debug('find: error getting client', error);
log('find: error getting client', error);
return null;
}
@@ -132,7 +132,7 @@ class StorageAdapter {
if (client.appId) {
const [appError, app] = await safe(apps.get(client.appId));
if (appError || !app) {
debug(`find: Unknown app for client with appId ${client.appId}`);
log(`find: Unknown app for client with appId ${client.appId}`);
return null;
}
@@ -183,12 +183,12 @@ class StorageAdapter {
}
async findByUserCode(userCode) {
debug(`[${this.name}] FIXME findByUserCode userCode:${userCode}`);
log(`[${this.name}] FIXME findByUserCode userCode:${userCode}`);
}
// this is called only on Session store. there is a payload.uid
async findByUid(uid) {
debug(`[${this.name}] findByUid: ${uid}`);
log(`[${this.name}] findByUid: ${uid}`);
const data = await StorageAdapter.getData(this.name);
for (const d in data) {
@@ -199,19 +199,19 @@ class StorageAdapter {
}
async consume(id) {
debug(`[${this.name}] consume: ${id}`);
log(`[${this.name}] consume: ${id}`);
await StorageAdapter.updateData(this.name, (data) => data[id].consumed = true);
}
async destroy(id) {
debug(`[${this.name}] destroy: ${id}`);
log(`[${this.name}] destroy: ${id}`);
await StorageAdapter.updateData(this.name, (data) => delete data[id]);
}
async revokeByGrantId(grantId) {
debug(`[${this.name}] revokeByGrantId: ${grantId}`);
log(`[${this.name}] revokeByGrantId: ${grantId}`);
await StorageAdapter.updateData(this.name, (data) => {
for (const d in data) {
@@ -256,7 +256,7 @@ async function consumeAuthCode(authCode) {
// This exposed to run on a cron job
async function cleanupExpired() {
debug('cleanupExpired');
log('cleanupExpired');
const types = [ 'AuthorizationCode', 'AccessToken', 'Grant', 'Interaction', 'RefreshToken', 'Session' ];
for (const type of types) {
@@ -282,7 +282,7 @@ async function renderError(error) {
language: await settings.get(settings.LANGUAGE_KEY),
};
debug('renderError: %o', error);
log('renderError: %o', error);
return ejs.render(TEMPLATE_ERROR, data);
}
@@ -351,7 +351,7 @@ async function interactionLogin(req, res, next) {
const ip = req.headers['x-forwarded-for'] || req.socket.remoteAddress || null;
const clientId = details.params.client_id;
debug(`interactionLogin: for OpenID client ${clientId} from ${ip}`);
log(`interactionLogin: for OpenID client ${clientId} from ${ip}`);
if (req.body.autoLoginToken) { // auto login for first admin/owner
if (typeof req.body.autoLoginToken !== 'string') return next(new HttpError(400, 'autoLoginToken must be string if provided'));
@@ -394,10 +394,10 @@ async function interactionLogin(req, res, next) {
if (userPasskeys.length > 0) {
const [passkeyError] = await safe(passkeys.verifyAuthentication(user, passkeyResponse));
if (passkeyError) {
debug(`interactionLogin: passkey verification failed for ${username}: ${passkeyError.message}`);
log(`interactionLogin: passkey verification failed for ${username}: ${passkeyError.message}`);
return next(new HttpError(401, 'Invalid passkey'));
}
debug(`interactionLogin: passkey verified for ${username}`);
log(`interactionLogin: passkey verified for ${username}`);
}
}
@@ -446,7 +446,7 @@ async function interactionConfirm(req, res, next) {
if (detailsError) return next(new HttpError(detailsError.statusCode, detailsError.error_description));
const { grantId, uid, prompt: { name, details }, params, session: { accountId }, lastSubmission } = interactionDetails;
debug(`route interaction confirm post uid:${uid} prompt.name:${name} accountId:${accountId}`);
log(`route interaction confirm post uid:${uid} prompt.name:${name} accountId:${accountId}`);
const client = await oidcClients.get(params.client_id);
if (!client) return next(new Error('Client not found'));
@@ -510,7 +510,7 @@ async function interactionConfirm(req, res, next) {
const auditSource = AuditSource.fromOidcRequest(req);
await eventlog.add(user.ghost ? eventlog.ACTION_USER_LOGIN_GHOST : eventlog.ACTION_USER_LOGIN, auditSource, { userId: user.id, user: users.removePrivateFields(user), appId: client.appId || null });
await safe(users.notifyLoginLocation(user, ip, userAgent, auditSource), { debug });
await safe(users.notifyLoginLocation(user, ip, userAgent, auditSource), { debug: log });
const result = { consent };
await gOidcProvider.interactionFinished(req, res, result, { mergeWithLastSubmission: true });
@@ -586,31 +586,31 @@ async function start() {
let keyEdDsa = await blobs.getString(blobs.OIDC_KEY_EDDSA);
if (!keyEdDsa) {
debug('Generating new OIDC EdDSA key');
log('Generating new OIDC EdDSA key');
const { privateKey } = await jose.generateKeyPair('EdDSA', { extractable: true });
keyEdDsa = Object.assign(await jose.exportJWK(privateKey), { alg: 'EdDSA' }); // alg is optional, but wp requires it
await blobs.setString(blobs.OIDC_KEY_EDDSA, JSON.stringify(keyEdDsa));
jwksKeys.push(keyEdDsa);
} else {
debug('Using existing OIDC EdDSA key');
log('Using existing OIDC EdDSA key');
jwksKeys.push(JSON.parse(keyEdDsa));
}
let keyRs256 = await blobs.getString(blobs.OIDC_KEY_RS256);
if (!keyRs256) {
debug('Generating new OIDC RS256 key');
log('Generating new OIDC RS256 key');
const { privateKey } = await jose.generateKeyPair('RS256', { extractable: true });
keyRs256 = Object.assign(await jose.exportJWK(privateKey), { alg: 'RS256' }); // alg is optional, but wp requires it
await blobs.setString(blobs.OIDC_KEY_RS256, JSON.stringify(keyRs256));
jwksKeys.push(keyRs256);
} else {
debug('Using existing OIDC RS256 key');
log('Using existing OIDC RS256 key');
jwksKeys.push(JSON.parse(keyRs256));
}
let cookieSecret = await settings.get(settings.OIDC_COOKIE_SECRET_KEY);
if (!cookieSecret) {
debug('Generating new cookie secret');
log('Generating new cookie secret');
cookieSecret = crypto.randomBytes(256).toString('base64');
await settings.set(settings.OIDC_COOKIE_SECRET_KEY, cookieSecret);
}
@@ -725,7 +725,7 @@ async function start() {
const { subdomain, domain } = await dashboard.getLocation();
const fqdn = dns.fqdn(subdomain, domain);
debug(`start: create provider for ${fqdn} at ${ROUTE_PREFIX}`);
log(`start: create provider for ${fqdn} at ${ROUTE_PREFIX}`);
gOidcProvider = new Provider(`https://${fqdn}${ROUTE_PREFIX}`, configuration);

6
src/once.js
View File

@@ -1,12 +1,12 @@
import debugModule from 'debug';
import logger from './logger.js';
const debug = debugModule('box:once');
const { log, trace } = logger('once');
// https://github.com/isaacs/once/blob/main/LICENSE (ISC)
function once (fn) {
const f = function () {
if (f.called) {
debug(`${f.name} was already called, returning previous return value`);
log(`${f.name} was already called, returning previous return value`);
return f.value;
}
f.called = true;

14
src/openssl.js
View File

@@ -1,19 +1,19 @@
import assert from 'node:assert';
import BoxError from './boxerror.js';
import crypto from 'node:crypto';
import debugModule from 'debug';
import logger from './logger.js';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import safe from 'safetydance';
import shellModule from './shell.js';
const debug = debugModule('box:openssl');
const { log, trace } = logger('openssl');
const shell = shellModule('openssl');
async function generateKey(type) {
debug(`generateKey: generating new key for${type}`);
log(`generateKey: generating new key for${type}`);
if (type === 'rsa4096') {
return await shell.spawn('openssl', ['genrsa', '4096'], { encoding: 'utf8' });
@@ -63,7 +63,7 @@ async function createCsr(key, cn, altNames) {
// while we pass the CN anyways, subjectAltName takes precedence
const csrPem = await shell.spawn('openssl', ['req', '-new', '-key', keyFilePath, '-outform', 'PEM', '-subj', `/CN=${cn}`, '-config', opensslConfigFile], { encoding: 'utf8' });
await safe(fs.promises.rm(tmpdir, { recursive: true, force: true }));
debug(`createCsr: csr file created for ${cn}`);
log(`createCsr: csr file created for ${cn}`);
return csrPem; // inspect with openssl req -text -noout -in hostname.csr -inform pem
};
@@ -81,7 +81,7 @@ async function getCertificateDates(cert) {
const notAfterDate = new Date(notAfter);
const daysLeft = (notAfterDate - new Date())/(24 * 60 * 60 * 1000);
debug(`expiryDate: ${lines[2]} notBefore=${notBefore} notAfter=${notAfter} daysLeft=${daysLeft}`);
log(`expiryDate: ${lines[2]} notBefore=${notBefore} notAfter=${notAfter} daysLeft=${daysLeft}`);
return { startDate: notBeforeDate, endDate: notAfterDate };
}
@@ -106,7 +106,7 @@ async function generateCertificate(domain) {
const opensslConf = safe.fs.readFileSync('/etc/ssl/openssl.cnf', 'utf8');
const cn = domain;
debug(`generateCertificate: domain=${domain} cn=${cn}`);
log(`generateCertificate: domain=${domain} cn=${cn}`);
// SAN must contain all the domains since CN check is based on implementation if SAN is found. -checkhost also checks only SAN if present!
const opensslConfWithSan = `${opensslConf}\n[SAN]\nsubjectAltName=DNS:${domain},DNS:*.${cn}\n`;
@@ -169,7 +169,7 @@ async function generateDkimKey() {
}
async function generateDhparam() {
debug('generateDhparam: generating dhparams');
log('generateDhparam: generating dhparams');
return await shell.spawn('openssl', ['dhparam', '-dsaparam', '2048'], { encoding: 'utf8' });
}

20
src/passkeys.js
View File

@@ -3,7 +3,7 @@ import BoxError from './boxerror.js';
import crypto from 'node:crypto';
import dashboard from './dashboard.js';
import database from './database.js';
import debugModule from 'debug';
import logger from './logger.js';
import safe from 'safetydance';
import {
generateRegistrationOptions,
@@ -13,7 +13,7 @@ import {
} from '@simplewebauthn/server';
import _ from './underscore.js';
const debug = debugModule('box:passkeys');
const { log, trace } = logger('passkeys');
const PASSKEY_FIELDS = [ 'id', 'userId', 'credentialId', 'publicKey', 'counter', 'transports', 'name', 'creationTime', 'lastUsedTime' ].join(',');
@@ -151,7 +151,7 @@ async function getRegistrationOptions(user) {
storeChallenge(user.id, options.challenge);
debug(`getRegistrationOptions: generated for user ${user.id}`);
log(`getRegistrationOptions: generated for user ${user.id}`);
return options;
}
@@ -179,7 +179,7 @@ async function verifyRegistration(user, response, name) {
}));
if (error) {
debug(`verifyRegistration: verification failed for user ${user.id}:`, error);
log(`verifyRegistration: verification failed for user ${user.id}:`, error);
throw new BoxError(BoxError.INVALID_CREDENTIALS, 'Passkey verification failed');
}
@@ -200,7 +200,7 @@ async function verifyRegistration(user, response, name) {
name || 'Passkey'
);
debug(`verifyRegistration: passkey registered for user ${user.id}`);
log(`verifyRegistration: passkey registered for user ${user.id}`);
return result;
}
@@ -224,7 +224,7 @@ async function getAuthenticationOptions(user) {
storeChallenge(user.id, options.challenge);
debug(`getAuthenticationOptions: generated for user ${user.id}`);
log(`getAuthenticationOptions: generated for user ${user.id}`);
return options;
}
@@ -243,12 +243,12 @@ async function verifyAuthentication(user, response) {
const passkey = await getByCredentialId(credentialIdBase64url);
if (!passkey) {
debug(`verifyAuthentication: passkey not found for credential ${credentialIdBase64url}`);
log(`verifyAuthentication: passkey not found for credential ${credentialIdBase64url}`);
throw new BoxError(BoxError.NOT_FOUND, 'Passkey not found');
}
if (passkey.userId !== user.id) {
debug(`verifyAuthentication: passkey belongs to different user`);
log(`verifyAuthentication: passkey belongs to different user`);
throw new BoxError(BoxError.INVALID_CREDENTIALS, 'Passkey does not belong to this user');
}
@@ -268,7 +268,7 @@ async function verifyAuthentication(user, response) {
}));
if (error) {
debug(`verifyAuthentication: verification failed for user ${user.id}:`, error);
log(`verifyAuthentication: verification failed for user ${user.id}:`, error);
throw new BoxError(BoxError.INVALID_CREDENTIALS, 'Passkey verification failed');
}
@@ -276,7 +276,7 @@ async function verifyAuthentication(user, response) {
await updateCounter(passkey.id, verification.authenticationInfo.newCounter);
debug(`verifyAuthentication: passkey verified for user ${user.id}`);
log(`verifyAuthentication: passkey verified for user ${user.id}`);
return { verified: true, passkeyId: passkey.id };
}

64
src/platform.js
View File

@@ -7,7 +7,7 @@ import constants from './constants.js';
import cron from './cron.js';
import dashboard from './dashboard.js';
import database from './database.js';
import debugModule from 'debug';
import logger from './logger.js';
import dockerProxy from './dockerproxy.js';
import fs from 'node:fs';
import infra from './infra_version.js';
@@ -26,7 +26,7 @@ import users from './users.js';
import volumes from './volumes.js';
import _ from './underscore.js';
const debug = debugModule('box:platform');
const { log, trace } = logger('platform');
const shell = shellModule('platform');
@@ -37,14 +37,14 @@ function getStatus() {
}
async function pruneVolumes() {
debug('pruneVolumes: remove all unused local volumes');
log('pruneVolumes: remove all unused local volumes');
const [error] = await safe(shell.spawn('docker', [ 'volume', 'prune', '--all', '--force' ], { encoding: 'utf8' }));
if (error) debug(`pruneVolumes: error pruning volumes: ${error.mesage}`);
if (error) log(`pruneVolumes: error pruning volumes: ${error.mesage}`);
}
async function createDockerNetwork() {
debug('createDockerNetwork: recreating docker network');
log('createDockerNetwork: recreating docker network');
await shell.spawn('docker', ['network', 'rm', '-f', 'cloudron'], {});
// the --ipv6 option will work even in ipv6 is disabled. fd00 is IPv6 ULA
@@ -53,13 +53,13 @@ async function createDockerNetwork() {
}
async function removeAllContainers() {
debug('removeAllContainers: removing all containers for infra upgrade');
log('removeAllContainers: removing all containers for infra upgrade');
const output = await shell.spawn('docker', ['ps', '-qa', '--filter', 'label=isCloudronManaged'], { encoding: 'utf8' });
if (!output) return;
for (const containerId of output.trim().split('\n')) {
debug(`removeAllContainers: stopping and removing ${containerId}`);
log(`removeAllContainers: stopping and removing ${containerId}`);
await shell.spawn('docker', ['stop', containerId], { encoding: 'utf8' });
await shell.spawn('docker', ['rm', '-f', containerId], { encoding: 'utf8' });
}
@@ -70,10 +70,10 @@ async function markApps(existingInfra, restoreOptions) {
assert.strictEqual(typeof restoreOptions, 'object'); // { skipDnsSetup }
if (existingInfra.version === 'none') { // cloudron is being restored from backup
debug('markApps: restoring apps');
log('markApps: restoring apps');
await apps.restoreApps(await apps.list(), restoreOptions, AuditSource.PLATFORM);
} else if (existingInfra.version !== infra.version) {
debug('markApps: reconfiguring apps');
log('markApps: reconfiguring apps');
reverseProxy.removeAppConfigs(); // should we change the cert location, nginx will not start
await apps.configureApps(await apps.list(), { scheduleNow: false }, AuditSource.PLATFORM); // we will schedule it when infra is ready
} else {
@@ -85,25 +85,25 @@ async function markApps(existingInfra, restoreOptions) {
if (changedAddons.length) {
// restart apps if docker image changes since the IP changes and any "persistent" connections fail
debug(`markApps: changedAddons: ${JSON.stringify(changedAddons)}`);
log(`markApps: changedAddons: ${JSON.stringify(changedAddons)}`);
await apps.restartAppsUsingAddons(changedAddons, AuditSource.PLATFORM);
} else {
debug('markApps: apps are already uptodate');
log('markApps: apps are already uptodate');
}
}
}
async function onInfraReady(infraChanged) {
debug(`onInfraReady: platform is ready. infra changed: ${infraChanged}`);
log(`onInfraReady: platform is ready. infra changed: ${infraChanged}`);
gStatus.message = 'Ready';
gStatus.state = 'ready';
if (infraChanged) await safe(pruneVolumes(), { debug }); // ignore error
if (infraChanged) await safe(pruneVolumes(), { debug: log }); // ignore error
await apps.schedulePendingTasks(AuditSource.PLATFORM);
await appTaskManager.start();
// only prune services on infra change (which starts services for upgrade)
if (infraChanged) safe(services.stopUnusedServices(), { debug });
if (infraChanged) safe(services.stopUnusedServices(), { debug: log });
}
async function startInfra(restoreOptions) {
@@ -111,7 +111,7 @@ async function startInfra(restoreOptions) {
if (constants.TEST && !process.env.TEST_CREATE_INFRA) return;
debug('startInfra: checking infrastructure');
log('startInfra: checking infrastructure');
let existingInfra = { version: 'none' };
if (fs.existsSync(paths.INFRA_VERSION_FILE)) {
@@ -121,13 +121,13 @@ async function startInfra(restoreOptions) {
// short-circuit for the restart case
if (_.isEqual(infra, existingInfra)) {
debug('startInfra: infra is uptodate at version %s', infra.version);
safe(services.applyServiceLimits(), { debug });
log('startInfra: infra is uptodate at version %s', infra.version);
safe(services.applyServiceLimits(), { debug: log });
await onInfraReady(false /* !infraChanged */);
return;
}
debug(`startInfra: updating infrastructure from ${existingInfra.version} to ${infra.version}`);
log(`startInfra: updating infrastructure from ${existingInfra.version} to ${infra.version}`);
for (let attempt = 0; attempt < 5; attempt++) {
try {
@@ -147,7 +147,7 @@ async function startInfra(restoreOptions) {
// for some reason, mysql arbitrary restarts making startup tasks fail. this makes the box update stuck
// LOST is when existing connection breaks. REFUSED is when new connection cannot connect at all
const retry = error.reason === BoxError.DATABASE_ERROR && (error.code === 'PROTOCOL_CONNECTION_LOST' || error.code === 'ECONNREFUSED');
debug(`startInfra: Failed to start services. retry=${retry} (attempt ${attempt}): ${error}`);
log(`startInfra: Failed to start services. retry=${retry} (attempt ${attempt}): ${error}`);
if (!retry) {
gStatus.message = `Failed to start services. ${error.stdout ?? ''} ${error.stderr ?? ''}`;
gStatus.state = 'failed';
@@ -163,7 +163,7 @@ async function startInfra(restoreOptions) {
async function onActivated(restoreOptions) {
assert.strictEqual(typeof restoreOptions, 'object'); // { skipDnsSetup }
debug('onActivated: starting post activation services');
log('onActivated: starting post activation services');
// Starting the infra after a user is available means:
// 1. mail bounces can now be sent to the cloudron owner
@@ -177,11 +177,11 @@ async function onActivated(restoreOptions) {
if (!constants.TEST) await timers.setTimeout(30000);
await reverseProxy.writeDefaultConfig({ activated :true });
debug('onActivated: finished');
log('onActivated: finished');
}
async function onDeactivated() {
debug('onDeactivated: stopping post activation services');
log('onDeactivated: stopping post activation services');
await cron.stopJobs();
await dockerProxy.stop();
@@ -189,7 +189,7 @@ async function onDeactivated() {
}
async function uninitialize() {
debug('uninitializing platform');
log('uninitializing platform');
if (await users.isActivated()) await onDeactivated();
@@ -201,7 +201,7 @@ async function onDashboardLocationSet(subdomain, domain) {
assert.strictEqual(typeof subdomain, 'string');
assert.strictEqual(typeof domain, 'string');
await safe(reverseProxy.writeDashboardConfig(subdomain, domain), { debug }); // ok to fail if no disk space
await safe(reverseProxy.writeDashboardConfig(subdomain, domain), { debug: log }); // ok to fail if no disk space
await oidcServer.stop();
await oidcServer.start();
@@ -210,7 +210,7 @@ async function onDashboardLocationSet(subdomain, domain) {
}
async function initialize() {
debug('initialize: start platform');
log('initialize: start platform');
await database.initialize();
await tasks.stopAllTasks(); // when box code crashes, systemd will clean up the control-group but not the tasks
@@ -225,13 +225,13 @@ async function initialize() {
// we remove the config as a simple security measure to not expose IP <-> domain
const activated = await users.isActivated();
if (!activated) {
debug('initialize: not activated. generating IP based redirection config');
await safe(reverseProxy.writeDefaultConfig({ activated: false }), { debug }); // ok to fail if no disk space
log('initialize: not activated. generating IP based redirection config');
await safe(reverseProxy.writeDefaultConfig({ activated: false }), { debug: log }); // ok to fail if no disk space
}
await updater.notifyBoxUpdate();
if (await users.isActivated()) safe(onActivated({ skipDnsSetup: false }), { debug }); // run in background
if (await users.isActivated()) safe(onActivated({ skipDnsSetup: false }), { debug: log }); // run in background
}
async function onDashboardLocationChanged(auditSource) {
@@ -239,9 +239,9 @@ async function onDashboardLocationChanged(auditSource) {
// mark all apps to be reconfigured, all have ExtraHosts injected
const [, installedApps] = await safe(apps.list());
await safe(apps.configureApps(installedApps, { scheduleNow: true }, auditSource), { debug });
await safe(apps.configureApps(installedApps, { scheduleNow: true }, auditSource), { debug: log });
await safe(services.rebuildService('turn', auditSource), { debug }); // to update the realm variable
await safe(services.rebuildService('turn', auditSource), { debug: log }); // to update the realm variable
}
async function onMailServerLocationChanged(auditSource) {
@@ -250,7 +250,7 @@ async function onMailServerLocationChanged(auditSource) {
// mark apps using email addon to be reconfigured
const [, installedApps] = await safe(apps.list());
const appsUsingEmail = installedApps.filter((a) => !!a.manifest.addons?.email || a.manifest.addons?.sendmail?.requiresValidCertificate);
await safe(apps.configureApps(appsUsingEmail, { scheduleNow: true }, auditSource), { debug });
await safe(apps.configureApps(appsUsingEmail, { scheduleNow: true }, auditSource), { debug: log });
}
async function onMailServerIncomingDomainsChanged(auditSource) {
@@ -259,7 +259,7 @@ async function onMailServerIncomingDomainsChanged(auditSource) {
// mark apps using email addon to be reconfigured
const [, installedApps] = await safe(apps.list());
const appsUsingEmail = installedApps.filter((a) => !!a.manifest.addons?.email);
await safe(apps.configureApps(appsUsingEmail, { scheduleNow: true }, auditSource), { debug });
await safe(apps.configureApps(appsUsingEmail, { scheduleNow: true }, auditSource), { debug: log });
}
export default {

30
src/provision.js
View File

@@ -6,7 +6,7 @@ import backuptask from './backuptask.js';
import BoxError from './boxerror.js';
import dashboard from './dashboard.js';
import constants from './constants.js';
import debugModule from 'debug';
import logger from './logger.js';
import dns from './dns.js';
import domains from './domains.js';
import eventlog from './eventlog.js';
@@ -27,7 +27,7 @@ import users from './users.js';
import tld from 'tldjs';
import tokens from './tokens.js';
const debug = debugModule('box:provision');
const { log, trace } = logger('provision');
// we cannot use tasks since the tasks table gets overwritten when db is imported
@@ -51,13 +51,13 @@ const gStatus = {
};
function setProgress(task, message) {
debug(`setProgress: ${task} - ${message}`);
log(`setProgress: ${task} - ${message}`);
gStatus[task].message = message;
}
async function ensureDhparams() {
if (fs.existsSync(paths.DHPARAMS_FILE)) return;
debug('ensureDhparams: generating dhparams');
log('ensureDhparams: generating dhparams');
const dhparams = await openssl.generateDhparam();
if (!safe.fs.writeFileSync(paths.DHPARAMS_FILE, dhparams)) throw new BoxError(BoxError.FS_ERROR, `Could not save dhparams.pem: ${safe.error.message}`);
}
@@ -75,7 +75,7 @@ async function setupTask(domain, auditSource) {
const location = { subdomain: constants.DASHBOARD_SUBDOMAIN, domain };
try {
debug(`setupTask: subdomain ${location.subdomain} and domain ${location.domain}`);
log(`setupTask: subdomain ${location.subdomain} and domain ${location.domain}`);
await dns.registerLocations([location], { overwriteDns: true }, (progress) => setProgress('setup', progress.message));
await dns.waitForLocations([location], (progress) => setProgress('setup', progress.message));
await reverseProxy.ensureCertificate(location, {}, auditSource);
@@ -85,7 +85,7 @@ async function setupTask(domain, auditSource) {
setProgress('setup', 'Done'),
await eventlog.add(eventlog.ACTION_PROVISION, auditSource, {});
} catch (error) {
debug('setupTask: error. %o', error);
log('setupTask: error. %o', error);
gStatus.setup.errorMessage = error.message;
}
@@ -111,7 +111,7 @@ async function setup(domainConfig, ipv4Config, ipv6Config, auditSource) {
const domain = domainConfig.domain.toLowerCase();
const zoneName = domainConfig.zoneName ? domainConfig.zoneName : (tld.getDomain(domain) || domain);
debug(`setup: domain ${domain} and zone ${zoneName}`);
log(`setup: domain ${domain} and zone ${zoneName}`);
const data = {
zoneName: zoneName,
@@ -127,9 +127,9 @@ async function setup(domainConfig, ipv4Config, ipv6Config, auditSource) {
await network.setIPv4Config(ipv4Config);
await network.setIPv6Config(ipv6Config);
safe(setupTask(domain, auditSource), { debug }); // now that args are validated run the task in the background
safe(setupTask(domain, auditSource), { debug: log }); // now that args are validated run the task in the background
} catch (error) {
debug('setup: error. %o', error);
log('setup: error. %o', error);
gStatus.setup.active = false;
gStatus.setup.errorMessage = error.message;
throw error;
@@ -144,7 +144,7 @@ async function activate(username, password, email, displayName, ip, auditSource)
assert.strictEqual(typeof ip, 'string');
assert.strictEqual(typeof auditSource, 'object');
debug(`activate: user: ${username} email:${email}`);
log(`activate: user: ${username} email:${email}`);
await appstore.registerCloudron3();
@@ -157,7 +157,7 @@ async function activate(username, password, email, displayName, ip, auditSource)
await eventlog.add(eventlog.ACTION_ACTIVATE, auditSource, {});
safe(platform.onActivated({ skipDnsSetup: false }), { debug }); // background
safe(platform.onActivated({ skipDnsSetup: false }), { debug: log }); // background
return {
userId: ownerId,
@@ -215,11 +215,11 @@ async function restoreTask(backupSite, remotePath, ipv4Config, ipv6Config, optio
await appstore.checkSubscription(); // never throws. worst case, user has to visit the Account view to refresh subscription info
safe(platform.onActivated({ skipDnsSetup: options.skipDnsSetup }), { debug }); // background
safe(platform.onActivated({ skipDnsSetup: options.skipDnsSetup }), { debug: log }); // background
await backupSites.storageApi(backupSite).teardown(backupSite.config);
} catch (error) {
debug('restoreTask: error. %o', error);
log('restoreTask: error. %o', error);
gStatus.restore.errorMessage = error ? error.message : '';
}
gStatus.restore.active = false;
@@ -260,9 +260,9 @@ async function restore(backupConfig, remotePath, version, ipv4Config, ipv6Config
const ipv6Error = await network.testIPv6Config(ipv6Config);
if (ipv6Error) throw ipv6Error;
safe(restoreTask(backupSite, remotePath, ipv4Config, ipv6Config, options, auditSource), { debug }); // now that args are validated run the task in the background
safe(restoreTask(backupSite, remotePath, ipv4Config, ipv6Config, options, auditSource), { debug: log }); // now that args are validated run the task in the background
} catch (error) {
debug('restore: error. %o', error);
log('restore: error. %o', error);
gStatus.restore.active = false;
gStatus.restore.errorMessage = error.message;
throw error;

10
src/proxyauth.js
View File

@@ -3,7 +3,7 @@ import assert from 'node:assert';
import blobs from './blobs.js';
import constants from './constants.js';
import dashboard from './dashboard.js';
import debugModule from 'debug';
import logger from './logger.js';
import ejs from 'ejs';
import express from 'express';
import fs from 'node:fs';
@@ -21,7 +21,7 @@ import settings from './settings.js';
import users from './users.js';
import util from 'node:util';
const debug = debugModule('box:proxyAuth');
const { log, trace } = logger('proxyAuth');
// heavily inspired from https://gock.net/blog/2020/nginx-subrequest-authentication-server/ and https://github.com/andygock/auth-server
@@ -36,7 +36,7 @@ function jwtVerify(req, res, next) {
jwt.verify(token, gTokenSecret, function (error, decoded) {
if (error) {
debug('jwtVerify: malformed token or bad signature', error.message);
log('jwtVerify: malformed token or bad signature', error.message);
req.user = null;
} else {
req.user = decoded.user || null;
@@ -160,7 +160,7 @@ async function login(req, res, next) {
async function callback(req, res, next) {
if (typeof req.query.code !== 'string') return next(new HttpError(400, 'missing query argument "code"'));
debug(`callback: with code ${req.query.code}`);
log(`callback: with code ${req.query.code}`);
const username = await oidcServer.consumeAuthCode(req.query.code);
if (!username) return next(new HttpError(400, 'invalid "code"'));
@@ -237,7 +237,7 @@ async function start() {
gTokenSecret = await blobs.getString(blobs.PROXY_AUTH_TOKEN_SECRET);
if (!gTokenSecret) {
debug('start: generating new token secret');
log('start: generating new token secret');
gTokenSecret = hat(64);
await blobs.setString(blobs.PROXY_AUTH_TOKEN_SECRET, gTokenSecret);
}

58
src/reverseproxy.js
View File

@@ -5,7 +5,7 @@ import blobs from './blobs.js';
import BoxError from './boxerror.js';
import constants from './constants.js';
import dashboard from './dashboard.js';
import debugModule from 'debug';
import logger from './logger.js';
import dns from './dns.js';
import docker from './docker.js';
import domains from './domains.js';
@@ -24,7 +24,7 @@ import settings from './settings.js';
import shellModule from './shell.js';
import tasks from './tasks.js';
const debug = debugModule('box:reverseproxy');
const { log, trace } = logger('reverseproxy');
const shell = shellModule('reverseproxy');
const NGINX_APPCONFIG_EJS = fs.readFileSync(import.meta.dirname + '/nginxconfig.ejs', { encoding: 'utf8' });
@@ -59,7 +59,7 @@ async function providerMatches(domainObject, cert) {
const mismatch = issuerMismatch || wildcardMismatch;
debug(`providerMatches: subject=${subject} domain=${domain} issuer=${issuer} `
log(`providerMatches: subject=${subject} domain=${domain} issuer=${issuer} `
+ `wildcard=${isWildcardCert}/${wildcard} prod=${isLetsEncryptProd}/${prod} `
+ `issuerMismatch=${issuerMismatch} wildcardMismatch=${wildcardMismatch} match=${!mismatch}`);
@@ -110,7 +110,7 @@ async function needsRenewal(cert, renewalInfo, options) {
let renew = false;
if (now.getTime() >= rt.getTime()) renew = true; // renew immediately since now is in the past
else if ((now.getTime() + (24*60*60*1000)) >= rt.getTime()) renew = true; // next cron run will be in the past
debug(`needsRenewal: ${renew}. ARI ${JSON.stringify(renewalInfo)}`);
log(`needsRenewal: ${renew}. ARI ${JSON.stringify(renewalInfo)}`);
return renew; // can wait
}
@@ -122,7 +122,7 @@ async function needsRenewal(cert, renewalInfo, options) {
isExpiring = (endDate - now) <= (30 * 24 * 60 * 60 * 1000); // expiring in a month
}
debug(`needsRenewal: ${isExpiring}. force: ${!!options.forceRenewal}`);
log(`needsRenewal: ${isExpiring}. force: ${!!options.forceRenewal}`);
return isExpiring;
}
@@ -185,7 +185,7 @@ async function setupTlsAddon(app) {
for (const content of contents) {
if (writeFileSync(`${certificateDir}/${content.filename}`, content.data)) ++changed;
}
debug(`setupTlsAddon: ${changed} files changed`);
log(`setupTlsAddon: ${changed} files changed`);
// clean up any certs of old locations
const filenamesInUse = new Set(contents.map(c => c.filename));
@@ -196,7 +196,7 @@ async function setupTlsAddon(app) {
safe.fs.unlinkSync(path.join(certificateDir, filename));
++removed;
}
debug(`setupTlsAddon: ${removed} files removed`);
log(`setupTlsAddon: ${removed} files removed`);
if (changed || removed) await docker.restartContainer(app.id);
}
@@ -215,7 +215,7 @@ async function setFallbackCertificate(domain, certificate) {
assert.strictEqual(typeof domain, 'string');
assert(certificate && typeof certificate === 'object');
debug(`setFallbackCertificate: setting certs for domain ${domain}`);
log(`setFallbackCertificate: setting certs for domain ${domain}`);
if (!safe.fs.writeFileSync(path.join(paths.NGINX_CERT_DIR, `${domain}.host.cert`), certificate.cert)) throw new BoxError(BoxError.FS_ERROR, safe.error.message);
if (!safe.fs.writeFileSync(path.join(paths.NGINX_CERT_DIR, `${domain}.host.key`), certificate.key)) throw new BoxError(BoxError.FS_ERROR, safe.error.message);
@@ -245,7 +245,7 @@ async function writeCertificate(location) {
const certFilePath = path.join(paths.NGINX_CERT_DIR, `${domain}.host.cert`);
const keyFilePath = path.join(paths.NGINX_CERT_DIR, `${domain}.host.key`);
debug(`writeCertificate: ${fqdn} will use fallback certs`);
log(`writeCertificate: ${fqdn} will use fallback certs`);
writeFileSync(certFilePath, domainObject.fallbackCertificate.cert);
writeFileSync(keyFilePath, domainObject.fallbackCertificate.key);
@@ -257,7 +257,7 @@ async function writeCertificate(location) {
let key = await blobs.getString(`${blobs.CERT_PREFIX}-${certName}.key`);
if (!key || !cert) { // use fallback certs if we didn't manage to get acme certs
debug(`writeCertificate: ${fqdn} will use fallback certs because acme is missing`);
log(`writeCertificate: ${fqdn} will use fallback certs because acme is missing`);
cert = domainObject.fallbackCertificate.cert;
key = domainObject.fallbackCertificate.key;
}
@@ -276,7 +276,7 @@ async function getKey(certName) {
const key = await blobs.getString(`${blobs.CERT_PREFIX}-${certName}.key`);
if (key) return key;
debug(`ensureKey: generating new key for ${certName}`);
log(`ensureKey: generating new key for ${certName}`);
// secp384r1 is same as prime256v1. openssl ecparam -list_curves. we used to use secp384r1 but it doesn't seem to be accepted by few mail servers
return await openssl.generateKey('secp256r1');
};
@@ -287,13 +287,13 @@ async function getRenewalInfo(cert, certName) {
if (Date.now() < (new Date(renewalInfo.valid)).getTime()) return renewalInfo; // still valid
debug(`getRenewalInfo: ${certName} refreshing`);
log(`getRenewalInfo: ${certName} refreshing`);
const [error, result] = await safe(acme2.getRenewalInfo(cert, renewalInfo.url));
if (error) {
debug(`getRenewalInfo: ${certName} error getting renewal info`, error);
log(`getRenewalInfo: ${certName} error getting renewal info`, error);
await blobs.del(`${blobs.CERT_PREFIX}-${certName}.renewal`);
} else {
debug(`getRenewalInfo: ${certName} updated: ${JSON.stringify(result)}`);
log(`getRenewalInfo: ${certName} updated: ${JSON.stringify(result)}`);
await blobs.setString(`${blobs.CERT_PREFIX}-${certName}.renewal`, JSON.stringify(result));
}
@@ -311,12 +311,12 @@ async function ensureCertificate(location, options, auditSource) {
const fqdn = dns.fqdn(location.subdomain, location.domain);
if (location.certificate) { // user certificate
debug(`ensureCertificate: ${fqdn} will use user certs`);
log(`ensureCertificate: ${fqdn} will use user certs`);
return;
}
if (domainObject.tlsConfig.provider === 'fallback') {
debug(`ensureCertificate: ${fqdn} will use fallback certs`);
log(`ensureCertificate: ${fqdn} will use fallback certs`);
return;
}
@@ -330,13 +330,13 @@ async function ensureCertificate(location, options, auditSource) {
const outdated = await needsRenewal(cert, renewalInfo, options);
if (sameProvider && !outdated) {
debug(`ensureCertificate: ${fqdn} acme cert exists and is up to date`);
log(`ensureCertificate: ${fqdn} acme cert exists and is up to date`);
return;
}
debug(`ensureCertificate: ${fqdn} acme cert exists but provider mismatch or needs renewal`);
log(`ensureCertificate: ${fqdn} acme cert exists but provider mismatch or needs renewal`);
}
debug(`ensureCertificate: ${fqdn} needs acme cert`);
log(`ensureCertificate: ${fqdn} needs acme cert`);
const [error, result] = await safe(acme2.getCertificate(fqdn, domainObject, key));
if (!error) {
@@ -346,7 +346,7 @@ async function ensureCertificate(location, options, auditSource) {
await blobs.setString(`${blobs.CERT_PREFIX}-${certName}.renewal`, JSON.stringify(result.renewalInfo));
}
debug(`ensureCertificate: error: ${error?.message || 'null'}`);
log(`ensureCertificate: error: ${error?.message || 'null'}`);
await safe(eventlog.add(eventlog.ACTION_CERTIFICATE_NEW, auditSource, { domain: fqdn, errorMessage: error?.message || '', renewalInfo: result?.renewalInfo || null }));
}
@@ -377,7 +377,7 @@ async function writeDashboardConfig(subdomain, domain) {
assert.strictEqual(typeof subdomain, 'string');
assert.strictEqual(typeof domain, 'string');
debug(`writeDashboardConfig: writing dashboard config for ${domain}`);
log(`writeDashboardConfig: writing dashboard config for ${domain}`);
const dashboardFqdn = dns.fqdn(subdomain, domain);
const location = { domain, fqdn: dashboardFqdn, certificate: null };
@@ -390,7 +390,7 @@ async function removeDashboardConfig(subdomain, domain) {
assert.strictEqual(typeof subdomain, 'string');
assert.strictEqual(typeof domain, 'string');
debug(`removeDashboardConfig: removing dashboard config of ${domain}`);
log(`removeDashboardConfig: removing dashboard config of ${domain}`);
const vhost = dns.fqdn(subdomain, domain);
const nginxConfigFilename = path.join(paths.NGINX_APPCONFIG_DIR, `dashboard/${vhost}.conf`);
@@ -473,7 +473,7 @@ async function writeAppLocationNginxConfig(app, location, certificatePath) {
const nginxConf = ejs.render(NGINX_APPCONFIG_EJS, data);
const filename = path.join(paths.NGINX_APPCONFIG_DIR, app.id, `${fqdn.replace('*', '_')}.conf`);
debug(`writeAppLocationNginxConfig: writing config for "${fqdn}" to ${filename} with options ${JSON.stringify(data)}`);
log(`writeAppLocationNginxConfig: writing config for "${fqdn}" to ${filename} with options ${JSON.stringify(data)}`);
writeFileSync(filename, nginxConf);
}
@@ -566,7 +566,7 @@ async function cleanupCerts(locations, auditSource, progressCallback) {
if (removedCertNames.length) await safe(eventlog.add(eventlog.ACTION_CERTIFICATE_CLEANUP, auditSource, { domains: removedCertNames }));
debug('cleanupCerts: done');
log('cleanupCerts: done');
}
async function checkCerts(options, auditSource, progressCallback) {
@@ -621,12 +621,12 @@ async function startRenewCerts(options, auditSource) {
assert.strictEqual(typeof auditSource, 'object');
const taskId = await tasks.add(tasks.TASK_CHECK_CERTS, [ options, auditSource ]);
safe(tasks.startTask(taskId, {}), { debug }); // background
safe(tasks.startTask(taskId, {}), { debug: log }); // background
return taskId;
}
function removeAppConfigs() {
debug('removeAppConfigs: removing app nginx configs');
log('removeAppConfigs: removing app nginx configs');
// remove all configs which are not the default or current dashboard
for (const entry of fs.readdirSync(paths.NGINX_APPCONFIG_DIR, { withFileTypes: true })) {
@@ -649,7 +649,7 @@ async function writeDefaultConfig(options) {
const keyFilePath = path.join(paths.NGINX_CERT_DIR, 'default.key');
if (!fs.existsSync(certFilePath) || !fs.existsSync(keyFilePath)) {
debug('writeDefaultConfig: create new cert');
log('writeDefaultConfig: create new cert');
const cn = 'cloudron-' + (new Date()).toISOString(); // randomize date a bit to keep firefox happy
@@ -672,7 +672,7 @@ async function writeDefaultConfig(options) {
const nginxConf = ejs.render(NGINX_APPCONFIG_EJS, data);
const nginxConfigFilename = path.join(paths.NGINX_APPCONFIG_DIR, constants.NGINX_DEFAULT_CONFIG_FILE_NAME);
debug(`writeDefaultConfig: writing configs for endpoint "${data.endpoint}"`);
log(`writeDefaultConfig: writing configs for endpoint "${data.endpoint}"`);
if (!safe.fs.writeFileSync(nginxConfigFilename, nginxConf)) throw new BoxError(BoxError.FS_ERROR, safe.error);
@@ -708,7 +708,7 @@ async function setTrustedIps(trustedIps) {
}
async function reprovision() {
debug('reprovision: restoring fallback certs and trusted ips');
log('reprovision: restoring fallback certs and trusted ips');
const result = await domains.list();

6
src/routes/accesscontrol.js
View File

@@ -1,13 +1,13 @@
import apps from '../apps.js';
import assert from 'node:assert';
import BoxError from '../boxerror.js';
import debugModule from 'debug';
import logger from '../logger.js';
import { HttpError } from '@cloudron/connect-lastmile';
import safe from 'safetydance';
import tokens from '../tokens.js';
import users from '../users.js';
const debug = debugModule('box:routes/accesscontrol');
const { log, trace } = logger('routes/accesscontrol');
async function passwordAuth(req, res, next) {
@@ -55,7 +55,7 @@ async function tokenAuth(req, res, next) {
const user = await users.get(token.identifier);
if (!user) return next(new HttpError(401, 'User not found'));
if (!user.active) {
debug(`tokenAuth: ${user.username || user.id} is not active`);
log(`tokenAuth: ${user.username || user.id} is not active`);
return next(new HttpError(401, 'User not active'));
}

8
src/routes/apps.js
View File

@@ -8,7 +8,7 @@ import backupSites from '../backupsites.js';
import BoxError from '../boxerror.js';
import community from '../community.js';
import constants from '../constants.js';
import debugModule from 'debug';
import logger from '../logger.js';
import { HttpError } from '@cloudron/connect-lastmile';
import { HttpSuccess } from '@cloudron/connect-lastmile';
import metrics from '../metrics.js';
@@ -18,7 +18,7 @@ import users from '../users.js';
import { getImageContentType } from '../image-content-type.js';
import WebSocket from 'ws';
const debug = debugModule('box:routes/apps');
const { log, trace } = logger('routes/apps');
async function load(req, res, next) {
@@ -829,7 +829,7 @@ async function startExecWebSocket(req, res, next) {
duplexStream.on('end', function () { ws.close(); });
duplexStream.on('close', function () { ws.close(); });
duplexStream.on('error', function (streamError) {
debug('duplexStream error: %o', streamError);
log('duplexStream error: %o', streamError);
});
duplexStream.on('data', function (data) {
if (ws.readyState !== WebSocket.OPEN) return;
@@ -837,7 +837,7 @@ async function startExecWebSocket(req, res, next) {
});
ws.on('error', function (wsError) {
debug('websocket error: %o', wsError);
log('websocket error: %o', wsError);
});
ws.on('message', function (msg) {
duplexStream.write(msg);

6
src/routes/auth.js
View File

@@ -2,7 +2,7 @@ import assert from 'node:assert';
import AuditSource from '../auditsource.js';
import BoxError from '../boxerror.js';
import constants from '../constants.js';
import debugModule from 'debug';
import logger from '../logger.js';
import eventlog from '../eventlog.js';
import { HttpError } from '@cloudron/connect-lastmile';
import { HttpSuccess } from '@cloudron/connect-lastmile';
@@ -12,7 +12,7 @@ import speakeasy from 'speakeasy';
import tokens from '../tokens.js';
import users from '../users.js';
const debug = debugModule('box:routes/cloudron');
const { log, trace } = logger('routes/cloudron');
async function login(req, res, next) {
@@ -32,7 +32,7 @@ async function login(req, res, next) {
const auditSource = AuditSource.fromRequest(req);
await eventlog.add(req.user.ghost ? eventlog.ACTION_USER_LOGIN_GHOST : eventlog.ACTION_USER_LOGIN, auditSource, { userId: req.user.id, user: users.removePrivateFields(req.user), type, appId: oidcClients.ID_CLI });
await safe(users.notifyLoginLocation(req.user, ip, userAgent, auditSource), { debug });
await safe(users.notifyLoginLocation(req.user, ip, userAgent, auditSource), { debug: log });
next(new HttpSuccess(200, token));
}

6
src/routes/mailserver.js
View File

@@ -1,7 +1,7 @@
import assert from 'node:assert';
import AuditSource from '../auditsource.js';
import BoxError from '../boxerror.js';
import debugModule from 'debug';
import logger from '../logger.js';
import http from 'node:http';
import { HttpError } from '@cloudron/connect-lastmile';
import { HttpSuccess } from '@cloudron/connect-lastmile';
@@ -9,7 +9,7 @@ import mailServer from '../mailserver.js';
import safe from 'safetydance';
import services from '../services.js';
const debug = debugModule('box:routes/mailserver');
const { log, trace } = logger('routes/mailserver');
async function proxyToMailContainer(port, pathname, req, res, next) {
@@ -61,7 +61,7 @@ async function proxyAndRestart(req, res, next) {
if (httpError) return next(httpError);
// for success, the proxy already sent the response. do not proceed to connect-lastmile which will result in double headers
await safe(mailServer.restart(), { debug });
await safe(mailServer.restart(), { debug: log });
});
}

26
src/routes/test/common.js
View File

@@ -1,7 +1,7 @@
import apps from '../../apps.js';
import appstore from '../../appstore.js';
import backupSites from '../../backupsites.js';
import debugModule from 'debug';
import logger from '../../logger.js';
import constants from '../../constants.js';
import database from '../../database.js';
import assert from 'node:assert/strict';
@@ -16,7 +16,7 @@ import tasks from '../../tasks.js';
import timers from 'timers/promises';
import tokens from '../../tokens.js';
const debug = debugModule('box:test/common');
const { log, trace } = logger('test/common');
const manifest = {
'id': 'io.cloudron.test',
@@ -98,17 +98,17 @@ const serverUrl = `http://localhost:${constants.PORT}`;
async function setupServer() {
debug('Setting up server');
log('Setting up server');
await database.initialize();
await database._clear();
await appstore._setApiServerOrigin(mockApiServerOrigin);
await oidcServer.stop();
await server.start();
debug('Set up server complete');
log('Set up server complete');
}
async function setup() {
debug('Setting up');
log('Setting up');
await setupServer();
@@ -165,15 +165,15 @@ async function setup() {
await settings._set(settings.APPSTORE_API_TOKEN_KEY, appstoreToken); // appstore token
debug('Setup complete');
log('Setup complete');
}
async function cleanup() {
debug('Cleaning up');
log('Cleaning up');
await server.stop();
await oidcServer.stop();
if (!nock.isActive()) nock.activate();
debug('Cleaned up');
log('Cleaned up');
}
function clearMailQueue() {
@@ -187,7 +187,7 @@ async function checkMails(number) {
}
async function waitForTask(taskId) {
debug(`Waiting for task: ${taskId}`);
log(`Waiting for task: ${taskId}`);
for (let i = 0; i < 30; i++) {
const result = await tasks.get(taskId);
@@ -197,7 +197,7 @@ async function waitForTask(taskId) {
throw new Error(`Task ${taskId} failed: ${result.error.message} - ${result.error.stack}`);
}
await timers.setTimeout(2000);
debug(`Waiting for task to ${taskId} finish`);
log(`Waiting for task to ${taskId} finish`);
}
throw new Error(`Task ${taskId} never finished`);
}
@@ -206,16 +206,16 @@ async function waitForAsyncTask(es) {
return new Promise((resolve, reject) => {
const messages = [];
es.addEventListener('message', function (message) {
debug(`waitForAsyncTask: ${message.data}`);
log(`waitForAsyncTask: ${message.data}`);
messages.push(JSON.parse(message.data));
if (messages[messages.length-1].type === 'done') {
debug('waitForAsyncTask: finished');
log('waitForAsyncTask: finished');
es.close();
resolve(messages);
}
});
es.addEventListener('error', function (error) {
debug('waitForAsyncTask: errored', error);
log('waitForAsyncTask: errored', error);
es.close();
const e = new Error(error.message);
e.code = error.code;

32
src/scheduler.js
View File

@@ -4,12 +4,12 @@ import BoxError from './boxerror.js';
import cloudron from './cloudron.js';
import constants from './constants.js';
import { CronJob } from 'cron';
import debugModule from 'debug';
import logger from './logger.js';
import docker from './docker.js';
import safe from 'safetydance';
import _ from './underscore.js';
const debug = debugModule('box:scheduler');
const { log, trace } = logger('scheduler');
const gState = {}; // appId -> { containerId, schedulerConfig (manifest+crontab), cronjobs }
@@ -17,12 +17,12 @@ const gSuspendedAppIds = new Set(); // suspended because some apptask is running
// TODO: this should probably also stop existing jobs to completely prevent race but the code is not re-entrant
function suspendAppJobs(appId) {
debug(`suspendAppJobs: ${appId}`);
log(`suspendAppJobs: ${appId}`);
gSuspendedAppIds.add(appId);
}
function resumeAppJobs(appId) {
debug(`resumeAppJobs: ${appId}`);
log(`resumeAppJobs: ${appId}`);
gSuspendedAppIds.delete(appId);
}
@@ -44,7 +44,7 @@ async function runTask(appId, taskName) {
if (!error && data?.State?.Running === true) {
const jobStartTime = new Date(data.State.StartedAt); // iso 8601
if ((new Date() - jobStartTime) < JOB_MAX_TIME) return;
debug(`runTask: ${containerName} is running too long, restarting`);
log(`runTask: ${containerName} is running too long, restarting`);
}
await docker.restartContainer(containerName);
@@ -65,10 +65,10 @@ async function createJobs(app, schedulerConfig) {
// stopJobs only deletes jobs since previous sync. This means that when box code restarts, none of the containers
// are removed. The deleteContainer here ensures we re-create the cron containers with the latest config
await safe(docker.deleteContainer(containerName)); // ignore error
const [error] = await safe(docker.createSubcontainer(app, containerName, [ '/bin/sh', '-c', command ], {} /* options */), { debug });
const [error] = await safe(docker.createSubcontainer(app, containerName, [ '/bin/sh', '-c', command ], {} /* options */), { debug: log });
if (error && error.reason !== BoxError.ALREADY_EXISTS) continue;
debug(`createJobs: ${taskName} (${app.fqdn}) will run in container ${containerName}`);
log(`createJobs: ${taskName} (${app.fqdn}) will run in container ${containerName}`);
let cronTime;
if (schedule === '@service') {
@@ -82,7 +82,7 @@ async function createJobs(app, schedulerConfig) {
cronTime,
onTick: async () => {
const [taskError] = await safe(runTask(appId, taskName)); // put the app id in closure, so we don't use the outdated app object by mistake
if (taskError) debug(`could not run task ${taskName} : ${taskError.message}`);
if (taskError) log(`could not run task ${taskName} : ${taskError.message}`);
},
start: true,
timeZone: tz
@@ -105,15 +105,15 @@ async function deleteAppJobs(appId, appState) {
const containerName = `${appId}-${taskName}`;
const [error] = await safe(docker.deleteContainer(containerName));
if (error) debug(`deleteAppJobs: failed to delete task container with name ${containerName} : ${error.message}`);
if (error) log(`deleteAppJobs: failed to delete task container with name ${containerName} : ${error.message}`);
}
}
async function deleteJobs() {
for (const appId of Object.keys(gState)) {
debug(`deleteJobs: removing jobs of ${appId}`);
log(`deleteJobs: removing jobs of ${appId}`);
const [error] = await safe(deleteAppJobs(appId, gState[appId]));
if (error) debug(`deleteJobs: error stopping jobs of removed app ${appId}: ${error.message}`);
if (error) log(`deleteJobs: error stopping jobs of removed app ${appId}: ${error.message}`);
delete gState[appId];
}
}
@@ -125,12 +125,12 @@ async function sync() {
const allAppIds = allApps.map(app => app.id);
const removedAppIds = _.difference(Object.keys(gState), allAppIds);
if (removedAppIds.length !== 0) debug(`sync: stopping jobs of removed apps ${JSON.stringify(removedAppIds)}`);
if (removedAppIds.length !== 0) log(`sync: stopping jobs of removed apps ${JSON.stringify(removedAppIds)}`);
for (const appId of removedAppIds) {
debug(`sync: removing jobs of ${appId}`);
log(`sync: removing jobs of ${appId}`);
const [error] = await safe(deleteAppJobs(appId, gState[appId]));
if (error) debug(`sync: error stopping jobs of removed app ${appId}: ${error.message}`);
if (error) log(`sync: error stopping jobs of removed app ${appId}: ${error.message}`);
delete gState[appId];
}
@@ -143,10 +143,10 @@ async function sync() {
if (_.isEqual(appState.schedulerConfig, schedulerConfig) && appState.containerId === app.containerId) continue; // nothing changed
}
debug(`sync: clearing jobs of ${app.id} (${app.fqdn})`);
log(`sync: clearing jobs of ${app.id} (${app.fqdn})`);
const [error] = await safe(deleteAppJobs(app.id, appState));
if (error) debug(`sync: error stopping jobs of ${app.id} : ${error.message}`);
if (error) log(`sync: error stopping jobs of ${app.id} : ${error.message}`);
if (!schedulerConfig) { // updated app version removed scheduler addon
delete gState[app.id];

10
src/scripts/backupupload.js
View File

@@ -2,10 +2,10 @@
import backuptask from '../backuptask.js';
import database from '../database.js';
import debugModule from 'debug';
import logger from '../logger.js';
import safe from 'safetydance';
const debug = debugModule('box:backupupload');
const { log, trace } = logger('backupupload');
// --check is used by run-tests to verify sudo access works. Caller must set BOX_ENV (e.g. BOX_ENV=test).
if (process.argv[2] === '--check') {
@@ -18,7 +18,7 @@ const remotePath = process.argv[2];
const format = process.argv[3];
const dataLayoutString = process.argv[4];
debug(`Backing up ${dataLayoutString} to ${remotePath}`);
log(`Backing up ${dataLayoutString} to ${remotePath}`);
process.on('SIGTERM', function () {
process.exit(0);
@@ -26,7 +26,7 @@ process.on('SIGTERM', function () {
// this can happen when the backup task is terminated (not box code)
process.on('disconnect', function () {
debug('parent process died');
log('parent process died');
process.exit(0);
});
@@ -45,7 +45,7 @@ function throttledProgressCallback(msecs) {
await database.initialize();
const [uploadError, result] = await safe(backuptask.upload(remotePath, format, dataLayoutString, throttledProgressCallback(5000)));
debug('upload completed. error: %o', uploadError);
log('upload completed. error: %o', uploadError);
process.send({ result, errorMessage: uploadError?.message });

2
src/scripts/starttask.sh
View File

@@ -45,7 +45,7 @@ options="-p TimeoutStopSec=10s -p MemoryMax=${memory_limit_mb}M -p OOMScoreAdjus
# it seems systemd-run does not return the exit status of the process despite --wait but atleast it waits
if ! systemd-run --unit "${service_name}" --wait --uid=${id} --gid=${id} \
-p TimeoutStopSec=2s -p MemoryMax=${memory_limit_mb}M -p OOMScoreAdjust=${oom_score_adjust} --nice "${nice}" \
--setenv HOME=${HOME} --setenv USER=${SUDO_USER} --setenv DEBUG=box:* --setenv BOX_ENV=${BOX_ENV} --setenv NODE_ENV=production \
--setenv HOME=${HOME} --setenv USER=${SUDO_USER} --setenv BOX_ENV=${BOX_ENV} --setenv NODE_ENV=production \
"${task_worker}" "${task_id}" "${logfile}"; then
echo "Service ${service_name} failed to run" # this only happens if the path to task worker itself is wrong
fi

12
src/server.js
View File

@@ -1,7 +1,7 @@
import assert from 'node:assert';
import AuditSource from './auditsource.js';
import constants from './constants.js';
import debugModule from 'debug';
import logger from './logger.js';
import eventlog from './eventlog.js';
import express from 'express';
import http from 'node:http';
@@ -14,14 +14,14 @@ import users from './users.js';
import util from 'node:util';
import { WebSocketServer } from 'ws';
const debug = debugModule('box:server');
const { log, trace } = logger('server');
let gHttpServer = null;
function notFoundHandler(req, res, next) {
const cleanUrl = req.url.replace(/(access_token=)[^&]+/, '$1' + '<redacted>');
debug(`no such route: ${req.method} ${cleanUrl}`);
log(`no such route: ${req.method} ${cleanUrl}`);
return next(new HttpError(404, 'No such route'));
}
@@ -507,9 +507,9 @@ async function initializeExpressSync() {
async function start() {
assert(gHttpServer === null, 'Server is already up and running.');
debug('==========================================');
debug(` Cloudron ${constants.VERSION} `);
debug('==========================================');
log('==========================================');
log(` Cloudron ${constants.VERSION} `);
log('==========================================');
await platform.initialize();

210
src/services.js
View File

@@ -7,7 +7,7 @@ import branding from './branding.js';
import constants from './constants.js';
import crypto from 'node:crypto';
import dashboard from './dashboard.js';
import debugModule from 'debug';
import logger from './logger.js';
import dig from './dig.js';
import docker from './docker.js';
import eventlog from './eventlog.js';
@@ -31,7 +31,7 @@ import sftp from './sftp.js';
import shellModule from './shell.js';
import superagent from '@cloudron/superagent';
const debug = debugModule('box:services');
const { log, trace } = logger('services');
const shell = shellModule('services');
const SERVICE_STATUS_STARTING = 'starting';
@@ -169,7 +169,7 @@ async function startAppServices(app) {
const [error] = await safe(APP_SERVICES[addon].start(instance)); // assume addons name is service name
// error ignored because we don't want "start app" to error. use can fix it from Services
if (error) debug(`startAppServices: ${addon}:${instance}. %o`, error);
if (error) log(`startAppServices: ${addon}:${instance}. %o`, error);
}
}
@@ -182,7 +182,7 @@ async function stopAppServices(app) {
const [error] = await safe(APP_SERVICES[addon].stop(instance)); // assume addons name is service name
// error ignored because we don't want "start app" to error. use can fix it from Services
if (error) debug(`stopAppServices: ${addon}:${instance}. %o`, error);
if (error) log(`stopAppServices: ${addon}:${instance}. %o`, error);
}
}
@@ -190,11 +190,11 @@ async function waitForContainer(containerName, tokenEnvName) {
assert.strictEqual(typeof containerName, 'string');
assert.strictEqual(typeof tokenEnvName, 'string');
debug(`Waiting for ${containerName}`);
log(`Waiting for ${containerName}`);
const result = await getContainerDetails(containerName, tokenEnvName);
await promiseRetry({ times: 20, interval: 15000, debug }, async () => {
await promiseRetry({ times: 20, interval: 15000, debug: log }, async () => {
const [networkError, response] = await safe(superagent.get(`http://${result.ip}:3000/healthcheck?access_token=${result.token}`)
.timeout(20000)
.ok(() => true));
@@ -217,7 +217,7 @@ async function ensureServiceRunning(serviceName) {
if (error) throw new BoxError(BoxError.ADDONS_ERROR, `${serviceName} container not found`);
if (container.State?.Running) return;
debug(`ensureServiceRunning: starting ${serviceName}`);
log(`ensureServiceRunning: starting ${serviceName}`);
await docker.startContainer(serviceName);
if (tokenEnvNames[serviceName]) await waitForContainer(serviceName, tokenEnvNames[serviceName]);
@@ -232,22 +232,22 @@ async function stopUnusedServices() {
}
}
debug(`stopUnusedServices: used addons - ${[...usedAddons]}`);
log(`stopUnusedServices: used addons - ${[...usedAddons]}`);
for (const name of LAZY_SERVICES) {
if (usedAddons.has(name)) continue;
debug(`stopUnusedServices: stopping ${name} (no apps use it)`);
await safe(docker.stopContainer(name), { debug });
log(`stopUnusedServices: stopping ${name} (no apps use it)`);
await safe(docker.stopContainer(name), { debug: log });
}
}
async function exportDatabase(addon) {
assert.strictEqual(typeof addon, 'string');
debug(`exportDatabase: exporting ${addon}`);
log(`exportDatabase: exporting ${addon}`);
if (fs.existsSync(path.join(paths.ADDON_CONFIG_DIR, `exported-${addon}`))) {
debug(`exportDatabase: already exported addon ${addon} in previous run`);
log(`exportDatabase: already exported addon ${addon} in previous run`);
return;
}
@@ -257,12 +257,12 @@ async function exportDatabase(addon) {
if (!app.manifest.addons || !(addon in app.manifest.addons)) continue; // app doesn't use the addon
if (app.installationState === apps.ISTATE_ERROR) continue; // missing db causes crash in old app addon containers
debug(`exportDatabase: exporting addon ${addon} of app ${app.id}`);
log(`exportDatabase: exporting addon ${addon} of app ${app.id}`);
// eslint-disable-next-line no-use-before-define -- circular: ADDONS references setup fns, setup fns call exportDatabase
const [error] = await safe(ADDONS[addon].backup(app, app.manifest.addons[addon]));
if (error) {
debug(`exportDatabase: error exporting ${addon} of app ${app.id}. %o`, error);
log(`exportDatabase: error exporting ${addon} of app ${app.id}. %o`, error);
// for errored apps, we can ignore if export had an error
if (app.installationState === apps.ISTATE_ERROR) continue;
throw error;
@@ -279,19 +279,19 @@ async function exportDatabase(addon) {
async function importDatabase(addon) {
assert.strictEqual(typeof addon, 'string');
debug(`importDatabase: importing ${addon}`);
log(`importDatabase: importing ${addon}`);
const allApps = await apps.list();
for (const app of allApps) {
if (!app.manifest.addons || !(addon in app.manifest.addons)) continue; // app doesn't use the addon
debug(`importDatabase: importing addon ${addon} of app ${app.id}`);
log(`importDatabase: importing addon ${addon} of app ${app.id}`);
const [error] = await safe(importAppDatabase(app, addon)); // eslint-disable-line no-use-before-define
if (!error) continue;
debug(`importDatabase: error importing ${addon} of app ${app.id}. Marking as errored. %o`, error);
log(`importDatabase: error importing ${addon} of app ${app.id}. Marking as errored. %o`, error);
// FIXME: there is no way to 'repair' if we are here. we need to make a separate apptask that re-imports db
// not clear, if repair workflow should be part of addon or per-app
await safe(apps.update(app.id, { installationState: apps.ISTATE_ERROR, error: { message: error.message } }));
@@ -304,7 +304,7 @@ async function setupLocalStorage(app, options) {
assert.strictEqual(typeof app, 'object');
assert.strictEqual(typeof options, 'object');
debug('setupLocalStorage');
log('setupLocalStorage');
const volumeDataDir = await apps.getStorageDir(app);
@@ -316,7 +316,7 @@ async function clearLocalStorage(app, options) {
assert.strictEqual(typeof app, 'object');
assert.strictEqual(typeof options, 'object');
debug('clearLocalStorage');
log('clearLocalStorage');
const volumeDataDir = await apps.getStorageDir(app);
const [error] = await safe(shell.sudo([ CLEARVOLUME_CMD, volumeDataDir ], {}));
@@ -327,7 +327,7 @@ async function teardownLocalStorage(app, options) {
assert.strictEqual(typeof app, 'object');
assert.strictEqual(typeof options, 'object');
debug('teardownLocalStorage');
log('teardownLocalStorage');
const volumeDataDir = await apps.getStorageDir(app);
const [error] = await safe(shell.sudo([ RMVOLUME_CMD, volumeDataDir ], {}));
@@ -340,7 +340,7 @@ async function backupSqlite(app, options) {
assert.strictEqual(typeof app, 'object');
assert.strictEqual(typeof options, 'object');
debug('Backing up sqlite');
log('Backing up sqlite');
const volumeDataDir = await apps.getStorageDir(app);
@@ -372,7 +372,7 @@ async function restoreSqlite(app, options) {
assert.strictEqual(typeof app, 'object');
assert.strictEqual(typeof options, 'object');
debug('Restoring sqlite');
log('Restoring sqlite');
const volumeDataDir = await apps.getStorageDir(app);
@@ -408,7 +408,7 @@ async function setupPersistentDirs(app) {
if (!app.manifest.persistentDirs) return;
debug('setupPersistentDirs');
log('setupPersistentDirs');
for (const dir of app.manifest.persistentDirs) {
const hostDir = persistentDirHostPath(app.id, dir);
@@ -422,7 +422,7 @@ async function teardownPersistentDirs(app) {
if (!app.manifest.persistentDirs) return;
debug('teardownPersistentDirs');
log('teardownPersistentDirs');
for (const dir of app.manifest.persistentDirs) {
const hostDir = persistentDirHostPath(app.id, dir);
@@ -438,7 +438,7 @@ async function runBackupCommand(app) {
if (!app.manifest.backupCommand) return;
debug('runBackupCommand');
log('runBackupCommand');
const volumeDataDir = await apps.getStorageDir(app);
const dataDirMount = volumeDataDir ? `-v ${volumeDataDir}:/app/data` : '';
@@ -462,7 +462,7 @@ async function runRestoreCommand(app) {
if (!app.manifest.restoreCommand) return;
debug('runRestoreCommand');
log('runRestoreCommand');
const volumeDataDir = await apps.getStorageDir(app);
const dataDirMount = volumeDataDir ? `-v ${volumeDataDir}:/app/data` : '';
@@ -504,7 +504,7 @@ async function setupTurn(app, options) {
{ name: 'CLOUDRON_TURN_SECRET', value: turnSecret }
];
debug('Setting up TURN');
log('Setting up TURN');
await addonConfigs.set(app.id, 'turn', env);
}
@@ -513,7 +513,7 @@ async function teardownTurn(app, options) {
assert.strictEqual(typeof app, 'object');
assert.strictEqual(typeof options, 'object');
debug('Tearing down TURN');
log('Tearing down TURN');
await addonConfigs.unset(app.id, 'turn');
}
@@ -543,7 +543,7 @@ async function setupEmail(app, options) {
{ name: 'CLOUDRON_EMAIL_LDAP_MAILBOXES_BASE_DN', value: 'ou=mailboxes,dc=cloudron' }
];
debug('Setting up Email');
log('Setting up Email');
await addonConfigs.set(app.id, 'email', env);
}
@@ -552,7 +552,7 @@ async function teardownEmail(app, options) {
assert.strictEqual(typeof app, 'object');
assert.strictEqual(typeof options, 'object');
debug('Tearing down Email');
log('Tearing down Email');
await addonConfigs.unset(app.id, 'email');
}
@@ -574,7 +574,7 @@ async function setupLdap(app, options) {
{ name: 'CLOUDRON_LDAP_BIND_PASSWORD', value: hat(4 * 128) } // this is ignored
];
debug('Setting up LDAP');
log('Setting up LDAP');
await addonConfigs.set(app.id, 'ldap', env);
}
@@ -583,7 +583,7 @@ async function teardownLdap(app, options) {
assert.strictEqual(typeof app, 'object');
assert.strictEqual(typeof options, 'object');
debug('Tearing down LDAP');
log('Tearing down LDAP');
await addonConfigs.unset(app.id, 'ldap');
}
@@ -592,7 +592,7 @@ async function setupSendMail(app, options) {
assert.strictEqual(typeof app, 'object');
assert.strictEqual(typeof options, 'object');
debug('Setting up SendMail');
log('Setting up SendMail');
const disabled = app.manifest.addons.sendmail.optional && !app.enableMailbox;
if (disabled) return await addonConfigs.unset(app.id, 'sendmail');
@@ -616,7 +616,7 @@ async function setupSendMail(app, options) {
if (app.manifest.addons.sendmail.supportsDisplayName) env.push({ name: 'CLOUDRON_MAIL_FROM_DISPLAY_NAME', value: app.mailboxDisplayName });
debug('Setting sendmail addon config to %j', env);
log('Setting sendmail addon config to %j', env);
await addonConfigs.set(app.id, 'sendmail', env);
}
@@ -624,7 +624,7 @@ async function teardownSendMail(app, options) {
assert.strictEqual(typeof app, 'object');
assert.strictEqual(typeof options, 'object');
debug('Tearing down sendmail');
log('Tearing down sendmail');
await addonConfigs.unset(app.id, 'sendmail');
}
@@ -633,7 +633,7 @@ async function setupRecvMail(app, options) {
assert.strictEqual(typeof app, 'object');
assert.strictEqual(typeof options, 'object');
debug('setupRecvMail: setting up recvmail');
log('setupRecvMail: setting up recvmail');
if (!app.enableInbox) return await addonConfigs.unset(app.id, 'recvmail');
@@ -653,7 +653,7 @@ async function setupRecvMail(app, options) {
{ name: 'CLOUDRON_MAIL_TO_DOMAIN', value: app.inboxDomain },
];
debug('setupRecvMail: setting recvmail addon config to %j', env);
log('setupRecvMail: setting recvmail addon config to %j', env);
await addonConfigs.set(app.id, 'recvmail', env);
}
@@ -661,7 +661,7 @@ async function teardownRecvMail(app, options) {
assert.strictEqual(typeof app, 'object');
assert.strictEqual(typeof options, 'object');
debug('teardownRecvMail: tearing down recvmail');
log('teardownRecvMail: tearing down recvmail');
await addonConfigs.unset(app.id, 'recvmail');
}
@@ -685,7 +685,7 @@ async function startMysql(existingInfra) {
const upgrading = existingInfra.version !== 'none' && requiresUpgrade(existingInfra.images.mysql, image);
if (upgrading) {
debug('startMysql: mysql will be upgraded');
log('startMysql: mysql will be upgraded');
await exportDatabase('mysql');
}
@@ -711,11 +711,11 @@ async function startMysql(existingInfra) {
--cap-add SYS_NICE \
${readOnly} -v /tmp -v /run ${image} ${cmd}`;
debug('startMysql: stopping and deleting previous mysql container');
log('startMysql: stopping and deleting previous mysql container');
await docker.stopContainer('mysql');
await docker.deleteContainer('mysql');
debug('startMysql: starting mysql container');
log('startMysql: starting mysql container');
await shell.bash(runCmd, { encoding: 'utf8' });
if (!serviceConfig.recoveryMode) {
@@ -730,7 +730,7 @@ async function setupMySql(app, options) {
assert.strictEqual(typeof app, 'object');
assert.strictEqual(typeof options, 'object');
debug('Setting up mysql');
log('Setting up mysql');
await ensureServiceRunning('mysql');
@@ -770,7 +770,7 @@ async function setupMySql(app, options) {
);
}
debug('Setting mysql addon config to %j', env);
log('Setting mysql addon config to %j', env);
await addonConfigs.set(app.id, 'mysql', env);
}
@@ -813,7 +813,7 @@ async function backupMySql(app, options) {
const database = mysqlDatabaseName(app.id);
debug('Backing up mysql');
log('Backing up mysql');
const result = await getContainerDetails('mysql', 'CLOUDRON_MYSQL_TOKEN');
@@ -829,7 +829,7 @@ async function restoreMySql(app, options) {
const database = mysqlDatabaseName(app.id);
debug('restoreMySql');
log('restoreMySql');
const result = await getContainerDetails('mysql', 'CLOUDRON_MYSQL_TOKEN');
@@ -855,7 +855,7 @@ async function startPostgresql(existingInfra) {
const upgrading = existingInfra.version !== 'none' && requiresUpgrade(existingInfra.images.postgresql, image);
if (upgrading) {
debug('startPostgresql: postgresql will be upgraded');
log('startPostgresql: postgresql will be upgraded');
await exportDatabase('postgresql');
}
@@ -880,11 +880,11 @@ async function startPostgresql(existingInfra) {
--label isCloudronManaged=true \
${readOnly} -v /tmp -v /run ${image} ${cmd}`;
debug('startPostgresql: stopping and deleting previous postgresql container');
log('startPostgresql: stopping and deleting previous postgresql container');
await docker.stopContainer('postgresql');
await docker.deleteContainer('postgresql');
debug('startPostgresql: starting postgresql container');
log('startPostgresql: starting postgresql container');
await shell.bash(runCmd, { encoding: 'utf8' });
if (!serviceConfig.recoveryMode) {
@@ -899,7 +899,7 @@ async function setupPostgreSql(app, options) {
assert.strictEqual(typeof app, 'object');
assert.strictEqual(typeof options, 'object');
debug('Setting up postgresql');
log('Setting up postgresql');
await ensureServiceRunning('postgresql');
@@ -931,7 +931,7 @@ async function setupPostgreSql(app, options) {
{ name: 'CLOUDRON_POSTGRESQL_DATABASE', value: data.database }
];
debug('Setting postgresql addon config to %j', env);
log('Setting postgresql addon config to %j', env);
await addonConfigs.set(app.id, 'postgresql', env);
}
@@ -942,7 +942,7 @@ async function clearPostgreSql(app, options) {
const { database, username } = postgreSqlNames(app.id);
const locale = options.locale || 'C';
debug('Clearing postgresql');
log('Clearing postgresql');
const result = await getContainerDetails('postgresql', 'CLOUDRON_POSTGRESQL_TOKEN');
@@ -972,7 +972,7 @@ async function backupPostgreSql(app, options) {
assert.strictEqual(typeof app, 'object');
assert.strictEqual(typeof options, 'object');
debug('Backing up postgresql');
log('Backing up postgresql');
const { database } = postgreSqlNames(app.id);
@@ -986,7 +986,7 @@ async function restorePostgreSql(app, options) {
assert.strictEqual(typeof app, 'object');
assert.strictEqual(typeof options, 'object');
debug('Restore postgresql');
log('Restore postgresql');
const { database, username } = postgreSqlNames(app.id);
@@ -1008,7 +1008,7 @@ async function startMongodb(existingInfra) {
const upgrading = existingInfra.version !== 'none' && requiresUpgrade(existingInfra.images.mongodb, image);
if (upgrading) {
debug('startMongodb: mongodb will be upgraded');
log('startMongodb: mongodb will be upgraded');
await exportDatabase('mongodb');
}
@@ -1032,16 +1032,16 @@ async function startMongodb(existingInfra) {
--label isCloudronManaged=true \
${readOnly} -v /tmp -v /run ${image} ${cmd}`;
debug('startMongodb: stopping and deleting previous mongodb container');
log('startMongodb: stopping and deleting previous mongodb container');
await docker.stopContainer('mongodb');
await docker.deleteContainer('mongodb');
if (!await hasAVX()) {
debug('startMongodb: not starting mongodb because CPU does not have AVX');
log('startMongodb: not starting mongodb because CPU does not have AVX');
return;
}
debug('startMongodb: starting mongodb container');
log('startMongodb: starting mongodb container');
await shell.bash(runCmd, { encoding: 'utf8' });
if (!serviceConfig.recoveryMode) {
@@ -1056,7 +1056,7 @@ async function setupMongoDb(app, options) {
assert.strictEqual(typeof app, 'object');
assert.strictEqual(typeof options, 'object');
debug('Setting up mongodb');
log('Setting up mongodb');
await ensureServiceRunning('mongodb');
@@ -1095,7 +1095,7 @@ async function setupMongoDb(app, options) {
env.push({ name: 'CLOUDRON_MONGODB_OPLOG_URL', value : `mongodb://${data.username}:${data.password}@mongodb:27017/local?authSource=${data.database}` });
}
debug('Setting mongodb addon config to %j', env);
log('Setting mongodb addon config to %j', env);
await addonConfigs.set(app.id, 'mongodb', env);
}
@@ -1141,7 +1141,7 @@ async function backupMongoDb(app, options) {
assert.strictEqual(typeof app, 'object');
assert.strictEqual(typeof options, 'object');
debug('Backing up mongodb');
log('Backing up mongodb');
if (!await hasAVX()) throw new BoxError(BoxError.ADDONS_ERROR, 'Error backing up MongoDB. CPU has no AVX support');
@@ -1159,7 +1159,7 @@ async function restoreMongoDb(app, options) {
assert.strictEqual(typeof app, 'object');
assert.strictEqual(typeof options, 'object');
debug('restoreMongoDb');
log('restoreMongoDb');
if (!await hasAVX()) throw new BoxError(BoxError.ADDONS_ERROR, 'Error restoring mongodb. CPU has no AVX support');
@@ -1193,7 +1193,7 @@ async function startGraphite(existingInfra) {
const upgrading = existingInfra.version !== 'none' && requiresUpgrade(existingInfra.images.graphite, image);
if (upgrading) debug('startGraphite: graphite will be upgraded');
if (upgrading) log('startGraphite: graphite will be upgraded');
const readOnly = !serviceConfig.recoveryMode ? '--read-only' : '';
const cmd = serviceConfig.recoveryMode ? '/bin/bash -c \'echo "Debug mode. Sleeping" && sleep infinity\'' : '';
@@ -1213,13 +1213,13 @@ async function startGraphite(existingInfra) {
--label isCloudronManaged=true \
${readOnly} -v /tmp -v /run ${image} ${cmd}`;
debug('startGraphite: stopping and deleting previous graphite container');
log('startGraphite: stopping and deleting previous graphite container');
await docker.stopContainer('graphite');
await docker.deleteContainer('graphite');
if (upgrading) await shell.sudo([ RMADDONDIR_CMD, 'graphite' ], {});
debug('startGraphite: starting graphite container');
log('startGraphite: starting graphite container');
await shell.bash(runCmd, { encoding: 'utf8' });
if (existingInfra.version !== 'none' && existingInfra.images.graphite !== image) await docker.deleteImage(existingInfra.images.graphite);
@@ -1229,7 +1229,7 @@ async function setupProxyAuth(app, options) {
assert.strictEqual(typeof app, 'object');
assert.strictEqual(typeof options, 'object');
debug('Setting up proxyAuth');
log('Setting up proxyAuth');
const enabled = app.sso && app.manifest.addons && app.manifest.addons.proxyAuth;
@@ -1238,7 +1238,7 @@ async function setupProxyAuth(app, options) {
const env = [ { name: 'CLOUDRON_PROXY_AUTH', value: '1' } ];
await addonConfigs.set(app.id, 'proxyauth', env);
debug('Creating OpenID client for proxyAuth');
log('Creating OpenID client for proxyAuth');
const proxyAuthClientId = `${app.id}-proxyauth`;
const result = await oidcClients.get(proxyAuthClientId);
@@ -1263,7 +1263,7 @@ async function teardownProxyAuth(app, options) {
await addonConfigs.unset(app.id, 'proxyauth');
debug('Deleting OpenID client for proxyAuth');
log('Deleting OpenID client for proxyAuth');
const proxyAuthClientId = `${app.id}-proxyauth`;
@@ -1275,7 +1275,7 @@ async function setupDocker(app, options) {
assert.strictEqual(typeof app, 'object');
assert.strictEqual(typeof options, 'object');
debug('Setting up docker');
log('Setting up docker');
const env = [ { name: 'CLOUDRON_DOCKER_HOST', value: `tcp://172.18.0.1:${constants.DOCKER_PROXY_PORT}` } ];
await addonConfigs.set(app.id, 'docker', env);
@@ -1343,7 +1343,7 @@ async function setupRedis(app, options) {
if (inspectError) {
await shell.bash(runCmd, { encoding: 'utf8' });
} else { // fast path
debug(`Re-using existing redis container with state: ${JSON.stringify(result.State)}`);
log(`Re-using existing redis container with state: ${JSON.stringify(result.State)}`);
}
if (isAppRunning && !recoveryMode) await waitForContainer(`redis-${app.id}`, 'CLOUDRON_REDIS_TOKEN');
@@ -1353,7 +1353,7 @@ async function clearRedis(app, options) {
assert.strictEqual(typeof app, 'object');
assert.strictEqual(typeof options, 'object');
debug('Clearing redis');
log('Clearing redis');
const disabled = app.manifest.addons.redis.optional && !app.enableRedis;
if (disabled) return;
@@ -1377,7 +1377,7 @@ async function teardownRedis(app, options) {
if (error) throw new BoxError(BoxError.FS_ERROR, `Error removing redis data: ${error.message}`);
safe.fs.rmSync(path.join(paths.LOG_DIR, `redis-${app.id}`), { recursive: true, force: true });
if (safe.error) debug('teardownRedis: cannot cleanup logs: %o', safe.error);
if (safe.error) log('teardownRedis: cannot cleanup logs: %o', safe.error);
await addonConfigs.unset(app.id, 'redis');
}
@@ -1389,7 +1389,7 @@ async function backupRedis(app, options) {
const disabled = app.manifest.addons.redis.optional && !app.enableRedis;
if (disabled) return;
debug('Backing up redis');
log('Backing up redis');
const result = await getContainerDetails('redis-' + app.id, 'CLOUDRON_REDIS_TOKEN');
const request = http.request(`http://${result.ip}:3000/backup?access_token=${result.token}`, { method: 'POST' });
@@ -1412,11 +1412,11 @@ async function startRedis(existingInfra) {
if (upgrading) await backupRedis(app, {});
debug(`startRedis: stopping and deleting previous redis container ${redisName}`);
log(`startRedis: stopping and deleting previous redis container ${redisName}`);
await docker.stopContainer(redisName);
await docker.deleteContainer(redisName);
debug(`startRedis: starting redis container ${redisName}`);
log(`startRedis: starting redis container ${redisName}`);
await setupRedis(app, app.manifest.addons.redis); // starts the container
}
@@ -1432,7 +1432,7 @@ async function restoreRedis(app, options) {
const disabled = app.manifest.addons.redis.optional && !app.enableRedis;
if (disabled) return;
debug('Restoring redis');
log('Restoring redis');
const result = await getContainerDetails('redis-' + app.id, 'CLOUDRON_REDIS_TOKEN');
const request = http.request(`http://${result.ip}:3000/restore?access_token=${result.token}`, { method: 'POST' });
@@ -1445,7 +1445,7 @@ async function setupTls(app, options) {
assert.strictEqual(typeof options, 'object');
if (!safe.fs.mkdirSync(`${paths.PLATFORM_DATA_DIR}/tls/${app.id}`, { recursive: true })) {
debug('Error creating tls directory');
log('Error creating tls directory');
throw new BoxError(BoxError.FS_ERROR, safe.error.message);
}
}
@@ -1483,7 +1483,7 @@ async function statusDocker() {
async function restartDocker() {
const [error] = await safe(shell.sudo([ RESTART_SERVICE_CMD, 'docker' ], {}));
if (error) debug(`restartDocker: error restarting docker. ${error.message}`);
if (error) log(`restartDocker: error restarting docker. ${error.message}`);
}
async function statusUnbound() {
@@ -1493,13 +1493,13 @@ async function statusUnbound() {
const [digError, digResult] = await safe(dig.resolve('ipv4.api.cloudron.io', 'A', { timeout: 10000 }));
if (!digError && Array.isArray(digResult) && digResult.length !== 0) return { status: SERVICE_STATUS_ACTIVE };
debug('statusUnbound: unbound is up, but failed to resolve ipv4.api.cloudron.io . %o %j', digError, digResult);
log('statusUnbound: unbound is up, but failed to resolve ipv4.api.cloudron.io . %o %j', digError, digResult);
return { status: SERVICE_STATUS_STARTING };
}
async function restartUnbound() {
const [error] = await safe(shell.sudo([ RESTART_SERVICE_CMD, 'unbound' ], {}));
if (error) debug(`restartDocker: error restarting unbound. ${error.message}`);
if (error) log(`restartDocker: error restarting unbound. ${error.message}`);
}
async function statusNginx() {
@@ -1509,7 +1509,7 @@ async function statusNginx() {
async function restartNginx() {
const [error] = await safe(shell.sudo([ RESTART_SERVICE_CMD, 'nginx' ], {}));
if (error) debug(`restartNginx: error restarting unbound. ${error.message}`);
if (error) log(`restartNginx: error restarting unbound. ${error.message}`);
}
async function statusGraphite() {
@@ -1676,7 +1676,7 @@ async function getServiceLogs(id, options) {
throw new BoxError(BoxError.NOT_FOUND, 'Service not found');
}
debug(`getServiceLogs: getting logs for ${name}`);
log(`getServiceLogs: getting logs for ${name}`);
let cp;
@@ -1730,7 +1730,7 @@ async function applyMemoryLimit(id) {
memoryLimit = serviceConfig && serviceConfig.memoryLimit ? serviceConfig.memoryLimit : APP_SERVICES[name].defaultMemoryLimit;
} else if (SERVICES[name]) {
if (name === 'mongodb' && !await hasAVX()) {
debug('applyMemoryLimit: skipping mongodb because CPU does not have AVX');
log('applyMemoryLimit: skipping mongodb because CPU does not have AVX');
return;
}
@@ -1743,12 +1743,12 @@ async function applyMemoryLimit(id) {
if (LAZY_SERVICES.includes(name)) {
const [error, container] = await safe(docker.inspect(containerName));
if (error || !container.State?.Running) {
debug(`applyMemoryLimit: skipping ${containerName} (not running)`);
log(`applyMemoryLimit: skipping ${containerName} (not running)`);
return;
}
}
debug(`applyMemoryLimit: ${containerName} ${JSON.stringify(serviceConfig)}`);
log(`applyMemoryLimit: ${containerName} ${JSON.stringify(serviceConfig)}`);
await docker.update(containerName, memoryLimit);
}
@@ -1766,7 +1766,7 @@ async function applyServiceLimits() {
changed = true;
}
safe(applyMemoryLimit(id), { debug });
safe(applyMemoryLimit(id), { debug: log });
}
if (changed) await settings.setJson(settings.SERVICES_CONFIG_KEY, servicesConfig);
@@ -1783,7 +1783,7 @@ async function applyServiceLimits() {
await apps.update(app.id, { servicesConfig: app.servicesConfig });
}
safe(applyMemoryLimit(`redis:${app.id}`), { debug });
safe(applyMemoryLimit(`redis:${app.id}`), { debug: log });
}
}
@@ -1797,7 +1797,7 @@ async function startTurn(existingInfra) {
let turnSecret = await blobs.getString(blobs.ADDON_TURN_SECRET);
if (!turnSecret) {
debug('startTurn: generating turn secret');
log('startTurn: generating turn secret');
turnSecret = 'a' + crypto.randomBytes(15).toString('hex'); // prefix with a to ensure string starts with a letter
await blobs.setString(blobs.ADDON_TURN_SECRET, turnSecret);
}
@@ -1825,11 +1825,11 @@ async function startTurn(existingInfra) {
--label isCloudronManaged=true \
${readOnly} -v /tmp -v /run ${image} ${cmd}`;
debug('startTurn: stopping and deleting previous turn container');
log('startTurn: stopping and deleting previous turn container');
await docker.stopContainer('turn');
await docker.deleteContainer('turn');
debug('startTurn: starting turn container');
log('startTurn: starting turn container');
await shell.bash(runCmd, { encoding: 'utf8' });
if (existingInfra.version !== 'none' && existingInfra.images.turn !== image) await docker.deleteImage(existingInfra.images.turn);
@@ -1877,7 +1877,7 @@ async function rebuildService(id, auditSource) {
// nothing to rebuild for now.
}
safe(applyMemoryLimit(id), { debug }); // do this in background. ok to fail
safe(applyMemoryLimit(id), { debug: log }); // do this in background. ok to fail
await eventlog.add(eventlog.ACTION_SERVICE_REBUILD, auditSource, { id });
}
@@ -1912,13 +1912,13 @@ async function configureService(id, data, auditSource) {
throw new BoxError(BoxError.NOT_FOUND, 'No such service');
}
debug(`configureService: ${id} rebuild=${needsRebuild}`);
log(`configureService: ${id} rebuild=${needsRebuild}`);
// do this in background
if (needsRebuild) {
safe(rebuildService(id, auditSource), { debug });
safe(rebuildService(id, auditSource), { debug: log });
} else {
safe(applyMemoryLimit(id), { debug });
safe(applyMemoryLimit(id), { debug: log });
}
await eventlog.add(eventlog.ACTION_SERVICE_CONFIGURE, auditSource, { id, data });
@@ -1950,14 +1950,14 @@ async function startServices(existingInfra, progressCallback) {
await fn(existingInfra);
}
safe(applyServiceLimits(), { debug });
safe(applyServiceLimits(), { debug: log });
}
async function teardownOauth(app, options) {
assert.strictEqual(typeof app, 'object');
assert.strictEqual(typeof options, 'object');
debug('teardownOauth');
log('teardownOauth');
await addonConfigs.unset(app.id, 'oauth');
}
@@ -1968,7 +1968,7 @@ async function setupOidc(app, options) {
if (!app.sso) return;
debug('Setting up OIDC');
log('Setting up OIDC');
const oidcAddonClientId = `${app.id}-oidc`;
const [error, result] = await safe(oidcClients.get(oidcAddonClientId));
@@ -1993,7 +1993,7 @@ async function teardownOidc(app, options) {
assert.strictEqual(typeof app, 'object');
assert.strictEqual(typeof options, 'object');
debug('Tearing down OIDC');
log('Tearing down OIDC');
const oidcAddonClientId = `${app.id}-oidc`;
@@ -2047,7 +2047,7 @@ async function moveDataDir(app, targetVolumeId, targetVolumePrefix) {
const resolvedSourceDir = await apps.getStorageDir(app);
const resolvedTargetDir = await apps.getStorageDir(Object.assign({}, app, { storageVolumeId: targetVolumeId, storageVolumePrefix: targetVolumePrefix }));
debug(`moveDataDir: migrating data from ${resolvedSourceDir} to ${resolvedTargetDir}`);
log(`moveDataDir: migrating data from ${resolvedSourceDir} to ${resolvedTargetDir}`);
if (resolvedSourceDir !== resolvedTargetDir) {
const [error] = await safe(shell.sudo([ MV_VOLUME_CMD, resolvedSourceDir, resolvedTargetDir ], {}));
@@ -2203,12 +2203,12 @@ async function setupAddons(app, addons) {
if (!addons) return;
debug('setupAddons: Setting up %j', Object.keys(addons));
log('setupAddons: Setting up %j', Object.keys(addons));
for (const addon of Object.keys(addons)) {
if (!(addon in ADDONS)) throw new BoxError(BoxError.NOT_FOUND, `No such addon: ${addon}`);
debug(`setupAddons: setting up addon ${addon} with options ${JSON.stringify(addons[addon])}`);
log(`setupAddons: setting up addon ${addon} with options ${JSON.stringify(addons[addon])}`);
await ADDONS[addon].setup(app, addons[addon]);
}
@@ -2220,12 +2220,12 @@ async function teardownAddons(app, addons) {
if (!addons) return;
debug('teardownAddons: Tearing down %j', Object.keys(addons));
log('teardownAddons: Tearing down %j', Object.keys(addons));
for (const addon of Object.keys(addons)) {
if (!(addon in ADDONS)) throw new BoxError(BoxError.NOT_FOUND, `No such addon: ${addon}`);
debug(`teardownAddons: Tearing down addon ${addon} with options ${JSON.stringify(addons[addon])}`);
log(`teardownAddons: Tearing down addon ${addon} with options ${JSON.stringify(addons[addon])}`);
await ADDONS[addon].teardown(app, addons[addon]);
}
@@ -2237,7 +2237,7 @@ async function backupAddons(app, addons) {
if (!addons) return;
debug('backupAddons: backing up %j', Object.keys(addons));
log('backupAddons: backing up %j', Object.keys(addons));
for (const addon of Object.keys(addons)) {
if (!(addon in ADDONS)) throw new BoxError(BoxError.NOT_FOUND, `No such addon: ${addon}`);
@@ -2252,7 +2252,7 @@ async function clearAddons(app, addons) {
if (!addons) return;
debug('clearAddons: clearing %j', Object.keys(addons));
log('clearAddons: clearing %j', Object.keys(addons));
for (const addon of Object.keys(addons)) {
if (!(addon in ADDONS)) throw new BoxError(BoxError.NOT_FOUND, `No such addon: ${addon}`);
@@ -2268,7 +2268,7 @@ async function restoreAddons(app, addons) {
if (!addons) return;
debug('restoreAddons: restoring %j', Object.keys(addons));
log('restoreAddons: restoring %j', Object.keys(addons));
for (const addon of Object.keys(addons)) {
if (!(addon in ADDONS)) throw new BoxError(BoxError.NOT_FOUND, `No such addon: ${addon}`);

16
src/sftp.js
View File

@@ -3,7 +3,7 @@ import assert from 'node:assert';
import blobs from './blobs.js';
import BoxError from './boxerror.js';
import constants from './constants.js';
import debugModule from 'debug';
import logger from './logger.js';
import docker from './docker.js';
import hat from './hat.js';
import infra from './infra_version.js';
@@ -15,7 +15,7 @@ import services from './services.js';
import shellModule from './shell.js';
import volumes from './volumes.js';
const debug = debugModule('box:sftp');
const { log, trace } = logger('sftp');
const shell = shellModule('sftp');
const DEFAULT_MEMORY_LIMIT = 256 * 1024 * 1024;
@@ -29,7 +29,7 @@ async function ensureKeys() {
const privateKeyFile = path.join(paths.SFTP_KEYS_DIR, `ssh_host_${keyType}_key`);
if (!privateKey || !publicKey) {
debug(`ensureSecrets: generating new sftp keys of type ${keyType}`);
log(`ensureSecrets: generating new sftp keys of type ${keyType}`);
safe.fs.unlinkSync(publicKeyFile);
safe.fs.unlinkSync(privateKeyFile);
const [error] = await safe(shell.spawn('ssh-keygen', ['-m', 'PEM', '-t', keyType, '-f', `${paths.SFTP_KEYS_DIR}/ssh_host_${keyType}_key`, '-q', '-N', ''], {}));
@@ -48,7 +48,7 @@ async function ensureKeys() {
async function start(existingInfra) {
assert.strictEqual(typeof existingInfra, 'object');
debug('start: re-creating container');
log('start: re-creating container');
const serviceConfig = await services.getServiceConfig('sftp');
const image = infra.images.sftp;
@@ -70,7 +70,7 @@ async function start(existingInfra) {
const hostDir = await apps.getStorageDir(app), mountDir = `/mnt/app-${app.id}`; // see also sftp:userSearchSftp
if (hostDir === null || !safe.fs.existsSync(hostDir)) { // this can fail if external mount does not have permissions for yellowtent user
// do not create host path when cloudron is restoring. this will then create dir with root perms making restore logic fail
debug(`Ignoring app data dir ${hostDir} for ${app.id} since it does not exist`);
log(`Ignoring app data dir ${hostDir} for ${app.id} since it does not exist`);
continue;
}
@@ -84,7 +84,7 @@ async function start(existingInfra) {
if (mounts.isManagedProvider(volume.mountType)) continue; // skip managed volume. these are acessed via /mnt/volumes mount above
if (!safe.fs.existsSync(volume.hostPath)) {
debug(`Ignoring volume host path ${volume.hostPath} since it does not exist`);
log(`Ignoring volume host path ${volume.hostPath} since it does not exist`);
continue;
}
@@ -118,11 +118,11 @@ async function start(existingInfra) {
--label isCloudronManaged=true \
${readOnly} -v /tmp -v /run ${image} ${cmd}`;
debug('startSftp: stopping and deleting previous sftp container');
log('startSftp: stopping and deleting previous sftp container');
await docker.stopContainer('sftp');
await docker.deleteContainer('sftp');
debug('startSftp: starting sftp container');
log('startSftp: starting sftp container');
await shell.bash(runCmd, { encoding: 'utf8' });
if (existingInfra.version !== 'none' && existingInfra.images.sftp !== image) await docker.deleteImage(existingInfra.images.sftp);

20
src/shell.js
View File

@@ -1,12 +1,12 @@
import assert from 'node:assert';
import BoxError from './boxerror.js';
import child_process from 'node:child_process';
import debugModule from 'debug';
import logger from './logger.js';
import path from 'node:path';
import safe from 'safetydance';
import _ from './underscore.js';
const debug = debugModule('box:shell');
const { log, trace } = logger('shell');
function lineCount(buffer) {
assert(Buffer.isBuffer(buffer));
@@ -30,7 +30,7 @@ function spawn(tag, file, args, options) {
assert(Array.isArray(args));
assert.strictEqual(typeof options, 'object'); // note: spawn() has no encoding option of it's own
debug(`${tag}: ${file} ${args.join(' ').replace(/\n/g, '\\n')}`);
log(`${tag}: ${file} ${args.join(' ').replace(/\n/g, '\\n')}`);
const maxLines = options.maxLines || Number.MAX_SAFE_INTEGER;
const logger = options.logger || null;
@@ -79,26 +79,26 @@ function spawn(tag, file, args, options) {
e.timedOut = timedOut;
e.terminated = terminated;
debug(`${tag}: ${file} ${args.join(' ').replace(/\n/g, '\\n')} errored`, e);
log(`${tag}: ${file} ${args.join(' ').replace(/\n/g, '\\n')} errored`, e);
reject(e);
});
cp.on('error', function (error) { // when the command itself could not be started
debug(`${tag}: ${file} ${args.join(' ').replace(/\n/g, '\\n')} errored`, error);
log(`${tag}: ${file} ${args.join(' ').replace(/\n/g, '\\n')} errored`, error);
});
cp.terminate = function () {
terminated = true;
// many approaches to kill sudo launched process failed. we now have a sudo wrapper to kill the full tree
child_process.execFile('/usr/bin/sudo', [ KILL_CHILD_CMD, cp.pid, process.pid ], { encoding: 'utf8' }, (error, stdout, stderr) => {
if (error) debug(`${tag}: failed to kill children`, stdout, stderr);
else debug(`${tag}: terminated ${cp.pid}`, stdout, stderr);
if (error) log(`${tag}: failed to kill children`, stdout, stderr);
else log(`${tag}: terminated ${cp.pid}`, stdout, stderr);
});
};
abortSignal?.addEventListener('abort', () => {
debug(`${tag}: aborting ${cp.pid}`);
log(`${tag}: aborting ${cp.pid}`);
cp.terminate();
}, { once: true });
@@ -106,10 +106,10 @@ function spawn(tag, file, args, options) {
if (options.timeout) {
killTimerId = setTimeout(async () => {
debug(`${tag}: timedout`);
log(`${tag}: timedout`);
timedOut = true;
if (typeof options.onTimeout !== 'function') return cp.terminate();
await safe(options.onTimeout(), { debug });
await safe(options.onTimeout(), { debug: log });
}, options.timeout);
}

18
src/storage/filesystem.js
View File

@@ -1,7 +1,7 @@
import assert from 'node:assert';
import BoxError from '../boxerror.js';
import crypto from 'crypto';
import debugModule from 'debug';
import logger from '../logger.js';
import df from '../df.js';
import fs from 'node:fs';
import mounts from '../mounts.js';
@@ -11,7 +11,7 @@ import safe from 'safetydance';
import shellModule from '../shell.js';
import _ from '../underscore.js';
const debug = debugModule('box:storage/filesystem');
const { log, trace } = logger('storage/filesystem');
const shell = shellModule('filesystem');
@@ -111,7 +111,7 @@ async function download(config, remotePath) {
assert.strictEqual(typeof remotePath, 'string');
const fullRemotePath = path.join(getRootPath(config), remotePath);
debug(`download: ${fullRemotePath}`);
log(`download: ${fullRemotePath}`);
if (!safe.fs.existsSync(fullRemotePath)) throw new BoxError(BoxError.NOT_FOUND, `File not found: ${fullRemotePath}`);
@@ -208,7 +208,7 @@ async function copyInternal(config, fromPath, toPath, options, progressCallback)
safe.fs.unlinkSync(identityFilePath);
if (!remoteCopyError) return;
if (remoteCopyError.code === 255) throw new BoxError(BoxError.EXTERNAL_ERROR, `SSH connection error: ${remoteCopyError.message}`); // do not attempt fallback copy for ssh errors
debug('SSH remote copy failed, trying sshfs copy'); // this can happen for sshfs mounted windows server
log('SSH remote copy failed, trying sshfs copy'); // this can happen for sshfs mounted windows server
}
const [copyError] = await safe(shell.spawn('cp', [ cpOptions, fullFromPath, fullToPath ], {}));
@@ -271,7 +271,7 @@ async function removeDir(config, limits, remotePathPrefix, progressCallback) {
safe.fs.unlinkSync(identityFilePath);
if (!remoteRmError) return;
if (remoteRmError.code === 255) throw new BoxError(BoxError.EXTERNAL_ERROR, `SSH connection error: ${remoteRmError.message}`); // do not attempt fallback copy for ssh errors
debug('SSH remote rm failed, trying sshfs rm'); // this can happen for sshfs mounted windows server
log('SSH remote rm failed, trying sshfs rm'); // this can happen for sshfs mounted windows server
}
const [error] = await safe(shell.spawn('rm', [ '-rf', fullPathPrefix ], {}));
@@ -311,13 +311,13 @@ function mountObjectFromConfig(config) {
async function setup(config) {
assert.strictEqual(typeof config, 'object');
debug('setup: removing old storage configuration');
log('setup: removing old storage configuration');
if (!mounts.isManagedProvider(config._provider)) return;
const mount = mountObjectFromConfig(config);
await safe(mounts.removeMount(mount), { debug }); // ignore error
await safe(mounts.removeMount(mount), { debug: log }); // ignore error
debug('setup: setting up new storage configuration');
log('setup: setting up new storage configuration');
await mounts.tryAddMount(mount, { timeout: 10 }); // 10 seconds
}
@@ -326,7 +326,7 @@ async function teardown(config) {
if (!mounts.isManagedProvider(config._provider)) return;
await safe(mounts.removeMount(mountObjectFromConfig(config)), { debug }); // ignore error
await safe(mounts.removeMount(mountObjectFromConfig(config)), { debug: log }); // ignore error
}
async function verifyConfig({ id, provider, config }) {

18
src/storage/gcs.js
View File

@@ -2,13 +2,13 @@ import assert from 'node:assert';
import async from 'async';
import BoxError from '../boxerror.js';
import constants from '../constants.js';
import debugModule from 'debug';
import logger from '../logger.js';
import { Storage as GCS } from '@google-cloud/storage';
import path from 'node:path';
import safe from 'safetydance';
import _ from '../underscore.js';
const debug = debugModule('box:storage/gcs');
const { log, trace } = logger('storage/gcs');
function getBucket(config) {
@@ -50,7 +50,7 @@ async function upload(config, limits, remotePath) {
const fullRemotePath = path.join(config.prefix, remotePath);
debug(`Uploading to ${fullRemotePath}`);
log(`Uploading to ${fullRemotePath}`);
return {
createStream() { return getBucket(config).file(fullRemotePath).createWriteStream({ resumable: false }); },
@@ -91,7 +91,7 @@ async function download(config, remotePath) {
assert.strictEqual(typeof remotePath, 'string');
const fullRemotePath = path.join(config.prefix, remotePath);
debug(`Download ${fullRemotePath} starting`);
log(`Download ${fullRemotePath} starting`);
const file = getBucket(config).file(fullRemotePath);
return file.createReadStream();
@@ -124,7 +124,7 @@ async function copyInternal(config, fullFromPath, fullToPath, progressCallback)
assert.strictEqual(typeof progressCallback, 'function');
const [copyError] = await safe(getBucket(config).file(fullFromPath).copy(fullToPath));
if (copyError) debug('copyBackup: gcs copy error. %o', copyError);
if (copyError) log('copyBackup: gcs copy error. %o', copyError);
if (copyError && copyError.code === 404) throw new BoxError(BoxError.NOT_FOUND, 'Old backup not found');
if (copyError) throw new BoxError(BoxError.EXTERNAL_ERROR, copyError.message);
}
@@ -175,7 +175,7 @@ async function remove(config, remotePath) {
const fullRemotePath = path.join(config.prefix, remotePath);
const [error] = await safe(getBucket(config).file(fullRemotePath).delete());
if (error) debug('removeBackups: Unable to remove %s (%s). Not fatal.', fullRemotePath, error.message);
if (error) log('removeBackups: Unable to remove %s (%s). Not fatal.', fullRemotePath, error.message);
}
async function removeDir(config, limits, remotePathPrefix, progressCallback) {
@@ -226,11 +226,11 @@ async function verifyConfig({ id, provider, config }) {
const testFile = bucket.file(path.join(config.prefix, 'snapshot/cloudron-testfile'));
const [saveError] = await safe(testFile.save('testfilecontents', { resumable: false }));
if (saveError) { // with bad creds, it return status 500 and "Cannot call write after a stream was destroyed"
debug('testConfig: failed uploading cloudron-testfile. %o', saveError);
log('testConfig: failed uploading cloudron-testfile. %o', saveError);
if (saveError.code == 403 || saveError.code == 404) throw new BoxError(BoxError.BAD_FIELD, saveError.message);
throw new BoxError(BoxError.EXTERNAL_ERROR, saveError.message);
}
debug('testConfig: uploaded cloudron-testfile');
log('testConfig: uploaded cloudron-testfile');
const query = { prefix: path.join(config.prefix, 'snapshot'), autoPaginate: false, maxResults: 1 };
const [listError] = await safe(bucket.getFiles(query));
@@ -238,7 +238,7 @@ async function verifyConfig({ id, provider, config }) {
const [delError] = await safe(bucket.file(path.join(config.prefix, 'snapshot/cloudron-testfile')).delete());
if (delError) throw new BoxError(BoxError.EXTERNAL_ERROR, delError.message);
debug('testConfig: deleted cloudron-testfile');
log('testConfig: deleted cloudron-testfile');
return _.pick(config, ['projectId', 'credentials', 'bucket', 'prefix']);
}

20
src/storage/s3.js
View File

@@ -5,7 +5,7 @@ import { ConfiguredRetryStrategy } from '@smithy/util-retry';
import constants from '../constants.js';
import consumers from 'node:stream/consumers';
import crypto from 'node:crypto';
import debugModule from 'debug';
import logger from '../logger.js';
import http from 'node:http';
import https from 'node:https';
import { NodeHttpHandler } from '@smithy/node-http-handler';
@@ -17,7 +17,7 @@ import safe from 'safetydance';
import { Upload } from '@aws-sdk/lib-storage';
import _ from '../underscore.js';
const debug = debugModule('box:storage/s3');
const { log, trace } = logger('storage/s3');
function S3_NOT_FOUND(error) {
return error instanceof NoSuchKey || error instanceof NoSuchBucket;
@@ -93,8 +93,8 @@ function createS3Client(config, options) {
const client = constants.TEST ? new globalThis.S3Mock(clientConfig) : new S3(clientConfig);
// https://github.com/aws/aws-sdk-js-v3/issues/6761#issuecomment-2574480834
// client.middlewareStack.add((next, context) => async (args) => {
// debug('AWS SDK context', context.clientName, context.commandName);
// debug('AWS SDK request input', JSON.stringify(args.input));
// log('AWS SDK context', context.clientName, context.commandName);
// log('AWS SDK request input', JSON.stringify(args.input));
// const result = await next(args);
// console.log('AWS SDK request output:', result.output);
// return result;
@@ -174,7 +174,7 @@ async function upload(config, limits, remotePath) {
};
const managedUpload = constants.TEST ? new globalThis.S3MockUpload(options) : new Upload(options);
// managedUpload.on('httpUploadProgress', (progress) => debug(`Upload progress: ${JSON.stringify(progress)}`));
// managedUpload.on('httpUploadProgress', (progress) => log(`Upload progress: ${JSON.stringify(progress)}`));
uploadPromise = managedUpload.done();
return passThrough;
@@ -183,7 +183,7 @@ async function upload(config, limits, remotePath) {
if (!uploadPromise) return; // stream was never created
const [error/*,data*/] = await safe(uploadPromise);
if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, `Upload error: code: ${error.Code} message: ${error.message}`); // sometimes message is null
// debug(`Upload finished. ${JSON.stringify(data)}`); // { ETag, $metadata:{httpStatusCode,requestId,attempts,totalRetryDelay},Bucket,Key}
// log(`Upload finished. ${JSON.stringify(data)}`); // { ETag, $metadata:{httpStatusCode,requestId,attempts,totalRetryDelay},Bucket,Key}
}
};
}
@@ -245,7 +245,7 @@ class S3MultipartDownloadStream extends Readable {
if (S3_NOT_FOUND(error)) {
this.destroy(new BoxError(BoxError.NOT_FOUND, `Backup not found: ${this._path}`));
} else {
debug(`download: ${this._path} s3 stream error. %o`, error);
log(`download: ${this._path} s3 stream error. %o`, error);
this.destroy(new BoxError(BoxError.EXTERNAL_ERROR, `Error multipartDownload ${this._path}. ${formatError(error)}`));
}
}
@@ -443,7 +443,7 @@ async function cleanup(config, progressCallback) {
for (const multipartUpload of uploads.Uploads) {
if (Date.now() - new Date(multipartUpload.Initiated) < 3 * 24 * 60 * 60 * 1000) continue; // 3 days ago
progressCallback({ message: `Cleaning up multi-part upload uploadId:${multipartUpload.UploadId} key:${multipartUpload.Key}` });
await safe(s3.abortMultipartUpload({ Bucket: config.bucket, Key: multipartUpload.Key, UploadId: multipartUpload.UploadId }), { debug }); // ignore error
await safe(s3.abortMultipartUpload({ Bucket: config.bucket, Key: multipartUpload.Key, UploadId: multipartUpload.UploadId }), { debug: log }); // ignore error
}
}
@@ -535,7 +535,7 @@ async function copyInternal(config, fullFromPath, fullToPath, fileSize, progress
const s3 = createS3Client(config, { retryStrategy: RETRY_STRATEGY }); // https://docs.aws.amazon.com/sdkref/latest/guide/feature-retry-behavior.html
function throwError(error) {
if (error) debug(`copy: s3 copy error when copying ${fullFromPath}: ${error}`);
if (error) log(`copy: s3 copy error when copying ${fullFromPath}: ${error}`);
if (error && S3_NOT_FOUND(error)) throw new BoxError(BoxError.NOT_FOUND, `Old backup not found: ${fullFromPath}`);
if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, `Error copying ${fullFromPath} (${fileSize} bytes): ${error.Code || ''} ${error}`);
@@ -603,7 +603,7 @@ async function copyInternal(config, fullFromPath, fullToPath, fileSize, progress
UploadId: uploadId
};
progressCallback({ message: `Aborting multipart copy of ${fullFromPath}` });
await safe(s3.abortMultipartUpload(abortParams), { debug }); // ignore any abort errors
await safe(s3.abortMultipartUpload(abortParams), { debug: log }); // ignore any abort errors
return throwError(copyError);
}

14
src/syncer.js
View File

@@ -1,14 +1,14 @@
import assert from 'node:assert';
import BoxError from './boxerror.js';
import DataLayout from './datalayout.js';
import debugModule from 'debug';
import logger from './logger.js';
import fs from 'node:fs';
import path from 'node:path';
import readline from 'node:readline';
import safe from 'safetydance';
import util from 'node:util';
const debug = debugModule('box:syncer');
const { log, trace } = logger('syncer');
function readCache(cacheFile) {
@@ -72,13 +72,13 @@ async function sync(dataLayout, cacheFile) {
// if cache is missing or if we crashed/errored in previous run, start out empty
if (!safe.fs.existsSync(cacheFile)) {
debug(`sync: cache file ${cacheFile} is missing, starting afresh`);
log(`sync: cache file ${cacheFile} is missing, starting afresh`);
delQueue.push({ operation: 'removedir', path: '', reason: 'nocache' });
} else if (safe.fs.existsSync(newCacheFile)) {
debug(`sync: new cache file ${newCacheFile} exists. previous run crashed, starting afresh`);
log(`sync: new cache file ${newCacheFile} exists. previous run crashed, starting afresh`);
delQueue.push({ operation: 'removedir', path: '', reason: 'crash' });
} else {
debug(`sync: loading cache file ${cacheFile}`);
log(`sync: loading cache file ${cacheFile}`);
cache = readCache(cacheFile);
}
@@ -168,7 +168,7 @@ async function finalize(integrityMap, cacheFile) {
const newCacheFile = `${cacheFile}.new`, tempCacheFile = `${cacheFile}.tmp`;
debug(`finalize: patching in integrity information into ${cacheFile}`);
log(`finalize: patching in integrity information into ${cacheFile}`);
const tempCacheFd = safe.fs.openSync(tempCacheFile, 'w'); // truncates any existing file
if (tempCacheFd === -1) throw new BoxError(BoxError.FS_ERROR, 'Error opening temp cache file: ' + safe.error.message);
@@ -192,7 +192,7 @@ async function finalize(integrityMap, cacheFile) {
safe.fs.unlinkSync(cacheFile);
safe.fs.unlinkSync(newCacheFile);
if (!safe.fs.renameSync(tempCacheFile, cacheFile)) debug('Unable to save new cache file');
if (!safe.fs.renameSync(tempCacheFile, cacheFile)) log('Unable to save new cache file');
}
export default {

16
src/system.js
View File

@@ -4,7 +4,7 @@ import asynctask from './asynctask.js';
const { AsyncTask } = asynctask;
import backupSites from './backupsites.js';
import BoxError from './boxerror.js';
import debugModule from 'debug';
import logger from './logger.js';
import df from './df.js';
import docker from './docker.js';
import eventlog from './eventlog.js';
@@ -18,7 +18,7 @@ import safe from 'safetydance';
import shellModule from './shell.js';
import volumes from './volumes.js';
const debug = debugModule('box:system');
const { log, trace } = logger('system');
const shell = shellModule('system');
@@ -168,7 +168,7 @@ async function getFilesystems() {
}
async function checkDiskSpace() {
debug('checkDiskSpace: checking disk space');
log('checkDiskSpace: checking disk space');
const filesystems = await getFilesystems();
@@ -185,7 +185,7 @@ async function checkDiskSpace() {
}
}
debug(`checkDiskSpace: disk space checked. low disk space: ${markdownMessage || 'no'}`);
log(`checkDiskSpace: disk space checked. low disk space: ${markdownMessage || 'no'}`);
if (markdownMessage) {
const finalMessage = `One or more file systems are running low on space. Please increase the disk size at the earliest.\n\n${markdownMessage}`;
@@ -223,7 +223,7 @@ class FilesystemUsageTask extends AsyncTask {
if (type === 'ext4' || type === 'xfs') { // hdparm only works with block devices
this.emitProgress(percent, 'Calculating Disk Speed');
const [speedError, speed] = await safe(hdparm(filesystem, { abortSignal }));
if (speedError) debug(`hdparm error ${filesystem}: ${speedError.message}`);
if (speedError) log(`hdparm error ${filesystem}: ${speedError.message}`);
this.emitData({ speed: speedError ? -1 : speed });
} else {
this.emitData({ speed: -1 });
@@ -241,7 +241,7 @@ class FilesystemUsageTask extends AsyncTask {
content.usage = content.id === 'docker' ? dockerDf.LayersSize : dockerDf.Volumes.map((v) => v.UsageData.Size).reduce((a,b) => a + b, 0);
} else {
const [error, duResult] = await safe(du(content.path, { abortSignal }));
if (error) debug(`du error ${content.path}: ${error.message}`); // can happen if app is installing etc
if (error) log(`du error ${content.path}: ${error.message}`); // can happen if app is installing etc
content.usage = duResult || 0;
}
usage += content.usage;
@@ -266,7 +266,7 @@ async function reboot() {
await notifications.unpin(notifications.TYPE_REBOOT, {});
const [error] = await safe(shell.sudo([ REBOOT_CMD ], {}));
if (error) debug('reboot: could not reboot. %o', error);
if (error) log('reboot: could not reboot. %o', error);
}
async function getInfo() {
@@ -369,7 +369,7 @@ async function checkUbuntuVersion() {
}
async function runSystemChecks() {
debug('runSystemChecks: checking status');
log('runSystemChecks: checking status');
const checks = [
checkRebootRequired(),

28
src/tasks.js
View File

@@ -1,7 +1,7 @@
import assert from 'node:assert';
import BoxError from './boxerror.js';
import database from './database.js';
import debugModule from 'debug';
import logger from './logger.js';
import logs from './logs.js';
import mysql from 'mysql2';
import path from 'node:path';
@@ -10,7 +10,7 @@ import safe from 'safetydance';
import shellModule from './shell.js';
import _ from './underscore.js';
const debug = debugModule('box:tasks');
const { log, trace } = logger('tasks');
const shell = shellModule('tasks');
const ESTOPPED = 'stopped';
@@ -70,7 +70,7 @@ async function update(id, task) {
assert.strictEqual(typeof id, 'string');
assert.strictEqual(typeof task, 'object');
debug(`updating task ${id} with: ${JSON.stringify(task)}`);
log(`updating task ${id} with: ${JSON.stringify(task)}`);
const args = [], fields = [];
for (const k in task) {
@@ -92,7 +92,7 @@ async function setCompleted(id, task) {
assert.strictEqual(typeof id, 'string');
assert.strictEqual(typeof task, 'object');
debug(`setCompleted - ${id}: ${JSON.stringify(task)}`);
log(`setCompleted - ${id}: ${JSON.stringify(task)}`);
await update(id, Object.assign({ completed: true }, task));
}
@@ -145,7 +145,7 @@ async function stopTask(id) {
if (!gTasks[id]) throw new BoxError(BoxError.BAD_STATE, 'task is not active');
debug(`stopTask: stopping task ${id}`);
log(`stopTask: stopping task ${id}`);
await shell.sudo([ STOP_TASK_CMD, id, ], {}); // note: this is stopping the systemd-run task. the sudo will exit when this exits
}
@@ -155,7 +155,7 @@ async function startTask(id, options) {
assert.strictEqual(typeof options, 'object');
const logFile = options.logFile || `${paths.TASKS_LOG_DIR}/${id}.log`;
debug(`startTask - starting task ${id} with options ${JSON.stringify(options)}. logs at ${logFile}`);
log(`startTask - starting task ${id} with options ${JSON.stringify(options)}. logs at ${logFile}`);
const ac = new AbortController();
gTasks[id] = ac;
@@ -166,17 +166,17 @@ async function startTask(id, options) {
abortSignal: ac.signal,
timeout: options.timeout || 0,
onTimeout: async () => { // custom stop because kill won't do. the task is running in some other process tree
debug(`onTimeout: ${id}`);
log(`onTimeout: ${id}`);
await stopTask(id);
}
};
safe(update(id, { pending: false }), { debug }); // background. we have to create the cp immediately to prevent race with stopTask()
safe(update(id, { pending: false }), { debug: log }); // background. we have to create the cp immediately to prevent race with stopTask()
const [sudoError] = await safe(shell.sudo([ START_TASK_CMD, id, logFile, options.nice || 0, options.memoryLimit || 400, options.oomScoreAdjust || 0 ], sudoOptions));
if (!gTasks[id]) { // when box code is shutting down, don't update the task status as "crashed". see stopAllTasks()
debug(`startTask: ${id} completed as a result of box shutdown`);
log(`startTask: ${id} completed as a result of box shutdown`);
return null;
}
@@ -186,7 +186,7 @@ async function startTask(id, options) {
if (!task) return null; // task disappeared on us. this can happen when db got cleared in tests
if (task.completed) { // task completed. we can trust the db result
debug(`startTask: ${id} completed. error: %o`, task.error);
log(`startTask: ${id} completed. error: %o`, task.error);
if (task.error) throw task.error;
return task.result;
}
@@ -201,19 +201,19 @@ async function startTask(id, options) {
else if (sudoError.code === 50) taskError = { message:`Task ${id} crashed with code ${sudoError.code}`, code: ECRASHED };
else taskError = { message:`Task ${id} crashed with unknown code ${sudoError.code}`, code: ECRASHED };
debug(`startTask: ${id} done. error: %o`, taskError);
await safe(setCompleted(id, { error: taskError }), { debug });
log(`startTask: ${id} done. error: %o`, taskError);
await safe(setCompleted(id, { error: taskError }), { debug: log });
throw taskError;
}
async function stopAllTasks() {
const acs = Object.values(gTasks);
debug(`stopAllTasks: ${acs.length} tasks are running. sending abort signal`);
log(`stopAllTasks: ${acs.length} tasks are running. sending abort signal`);
gTasks = {}; // this signals startTask() to not set completion status as "crashed"
acs.forEach(ac => ac.abort()); // cleanup all the sudos and systemd-run
const [error] = await safe(shell.sudo([ STOP_TASK_CMD, 'all' ], { cwd: paths.baseDir() }));
if (error) debug(`stopAllTasks: error stopping stasks: ${error.message}`);
if (error) log(`stopAllTasks: error stopping stasks: ${error.message}`);
}
async function getLogs(task, options) {

16
src/taskworker.js
View File

@@ -19,7 +19,7 @@ import safe from 'safetydance';
import tasks from './tasks.js';
import timers from 'timers/promises';
import updater from './updater.js';
import debugModule from 'debug';
import logger from './logger.js';
const TASKS = { // indexed by task type
app: apptask.run,
@@ -100,28 +100,28 @@ async function main() {
return process.exit(50);
}
const debug = debugModule('box:taskworker'); // import this here so that logging handler is already setup
const { log, trace } = logger('taskworker'); // import this here so that logging handler is already setup
process.on('SIGTERM', () => {
debug('Terminated');
log('Terminated');
exitSync({ code: 70 });
});
// ensure we log task crashes with the task logs. neither console.log nor debug are sync for some reason
process.on('uncaughtException', (error) => exitSync({ error, code: 1 }));
debug(`Starting task ${taskId}. Logs are at ${logFile}`);
log(`Starting task ${taskId}. Logs are at ${logFile}`);
const [getError, task] = await safe(tasks.get(taskId));
if (getError) return exitSync({ error: getError, code: 50 });
if (!task) return exitSync({ error: new Error(`Task ${taskId} not found`), code: 50 });
async function progressCallback(progress) {
await safe(tasks.update(taskId, progress), { debug });
await safe(tasks.update(taskId, progress), { debug: log });
}
const taskName = task.type.replace(/_.*/,'');
debug(`Running task of type ${taskName}`);
log(`Running task of type ${taskName}`);
const [runError, result] = await safe(TASKS[taskName].apply(null, task.args.concat(progressCallback)));
const progress = {
result: result || null,
@@ -129,9 +129,9 @@ async function main() {
percent: 100
};
await safe(tasks.setCompleted(taskId, progress), { debug });
await safe(tasks.setCompleted(taskId, progress), { debug: log });
debug(`Task took ${(new Date() - startTime)/1000} seconds`);
log(`Task took ${(new Date() - startTime)/1000} seconds`);
exitSync({ error: runError, code: (!runError || runError instanceof BoxError) ? 0 : 50 }); // handled error vs run time crash
}

10
src/translations.js
View File

@@ -1,12 +1,12 @@
import assert from 'node:assert';
import cloudron from './cloudron.js';
import debugModule from 'debug';
import logger from './logger.js';
import fs from 'node:fs';
import path from 'node:path';
import paths from './paths.js';
import safe from 'safetydance';
const debug = debugModule('box:translation');
const { log, trace } = logger('translation');
// to be used together with getTranslations() => { translations, fallback }
@@ -45,13 +45,13 @@ function translate(input, assets) {
async function getTranslations() {
const fallbackData = fs.readFileSync(path.join(paths.TRANSLATIONS_DIR, 'en.json'), 'utf8');
if (!fallbackData) debug(`getTranslations: Fallback language en not found. ${safe.error.message}`);
if (!fallbackData) log(`getTranslations: Fallback language en not found. ${safe.error.message}`);
const fallback = safe.JSON.parse(fallbackData) || {};
const lang = await cloudron.getLanguage();
const translationData = safe.fs.readFileSync(path.join(paths.TRANSLATIONS_DIR, `${lang}.json`), 'utf8');
if (!translationData) debug(`getTranslations: Requested language ${lang} not found. ${safe.error.message}`);
if (!translationData) log(`getTranslations: Requested language ${lang} not found. ${safe.error.message}`);
const translations = safe.JSON.parse(translationData) || {};
return { translations, fallback };
@@ -60,7 +60,7 @@ async function getTranslations() {
async function listLanguages() {
const [error, result] = await safe(fs.promises.readdir(paths.TRANSLATIONS_DIR));
if (error) {
debug(`listLanguages: Failed to list translations. %${error.message}`);
log(`listLanguages: Failed to list translations. %${error.message}`);
return [ 'en' ]; // we always return english to avoid dashboard breakage
}

48
src/updater.js
View File

@@ -10,7 +10,7 @@ import constants from './constants.js';
import cron from './cron.js';
import { CronTime } from 'cron';
import crypto from 'node:crypto';
import debugModule from 'debug';
import logger from './logger.js';
import df from './df.js';
import eventlog from './eventlog.js';
import fs from 'node:fs';
@@ -26,7 +26,7 @@ import settings from './settings.js';
import shellModule from './shell.js';
import tasks from './tasks.js';
const debug = debugModule('box:updater');
const { log, trace } = logger('updater');
const shell = shellModule('updater');
@@ -58,11 +58,11 @@ async function downloadBoxUrl(url, file) {
safe.fs.unlinkSync(file);
await promiseRetry({ times: 10, interval: 5000, debug }, async function () {
debug(`downloadBoxUrl: downloading ${url} to ${file}`);
await promiseRetry({ times: 10, interval: 5000, debug: log }, async function () {
log(`downloadBoxUrl: downloading ${url} to ${file}`);
const [error] = await safe(shell.spawn('curl', ['-s', '--fail', url, '-o', file], { encoding: 'utf8' }));
if (error) throw new BoxError(BoxError.NETWORK_ERROR, `Failed to download ${url}: ${error.message}`);
debug('downloadBoxUrl: done');
log('downloadBoxUrl: done');
});
}
@@ -72,13 +72,13 @@ async function gpgVerifyBoxTarball(file, sig) {
const [error, stdout] = await safe(shell.spawn('/usr/bin/gpg', ['--status-fd', '1', '--no-default-keyring', '--keyring', RELEASES_PUBLIC_KEY, '--verify', sig, file], { encoding: 'utf8' }));
if (error) {
debug(`gpgVerifyBoxTarball: command failed. error: ${error}\n stdout: ${error.stdout}\n stderr: ${error.stderr}`);
log(`gpgVerifyBoxTarball: command failed. error: ${error}\n stdout: ${error.stdout}\n stderr: ${error.stderr}`);
throw new BoxError(BoxError.NOT_SIGNED, `The signature in ${path.basename(sig)} could not be verified (command failed)`);
}
if (stdout.indexOf('[GNUPG:] VALIDSIG 0EADB19CDDA23CD0FE71E3470A372F8703C493CC') !== -1) return; // success
debug(`gpgVerifyBoxTarball: verification of ${sig} failed: ${stdout}\n`);
log(`gpgVerifyBoxTarball: verification of ${sig} failed: ${stdout}\n`);
throw new BoxError(BoxError.NOT_SIGNED, `The signature in ${path.basename(sig)} could not be verified (bad sig)`);
}
@@ -87,13 +87,13 @@ async function extractBoxTarball(tarball, dir) {
assert.strictEqual(typeof tarball, 'string');
assert.strictEqual(typeof dir, 'string');
debug(`extractBoxTarball: extracting ${tarball} to ${dir}`);
log(`extractBoxTarball: extracting ${tarball} to ${dir}`);
const [error] = await safe(shell.spawn('tar', ['-zxf', tarball, '-C', dir], { encoding: 'utf8' }));
if (error) throw new BoxError(BoxError.FS_ERROR, `Failed to extract release package: ${error.message}`);
safe.fs.unlinkSync(tarball);
debug('extractBoxTarball: extracted');
log('extractBoxTarball: extracted');
}
async function verifyBoxUpdateInfo(versionsFile, updateInfo) {
@@ -115,7 +115,7 @@ async function downloadAndVerifyBoxRelease(updateInfo) {
const oldArtifactNames = filenames.filter(f => f.startsWith('box-'));
for (const artifactName of oldArtifactNames) {
const fullPath = path.join(os.tmpdir(), artifactName);
debug(`downloadAndVerifyBoxRelease: removing old artifact ${fullPath}`);
log(`downloadAndVerifyBoxRelease: removing old artifact ${fullPath}`);
await fs.promises.rm(fullPath, { recursive: true, force: true });
}
@@ -173,7 +173,7 @@ async function updateBox(boxUpdateInfo, options, progressCallback) {
await locks.wait(locks.TYPE_BOX_UPDATE);
debug(`Updating box with ${boxUpdateInfo.sourceTarballUrl}`);
log(`Updating box with ${boxUpdateInfo.sourceTarballUrl}`);
progressCallback({ percent: 70, message: 'Installing update...' });
const [error] = await safe(shell.sudo([ UPDATE_CMD, packageInfo.file, process.stdout.logFile ], {})); // run installer.sh from new box code as a separate service
if (error) {
@@ -226,9 +226,9 @@ async function startBoxUpdateTask(options, auditSource) {
// background
tasks.startTask(taskId, { timeout: 20 * 60 * 60 * 1000 /* 20 hours */, nice: 15, memoryLimit })
.then(() => debug('startBoxUpdateTask: update task completed'))
.then(() => log('startBoxUpdateTask: update task completed'))
.catch(async (updateError) => {
debug('Update failed with error. %o', updateError);
log('Update failed with error. %o', updateError);
await locks.release(locks.TYPE_BOX_UPDATE_TASK);
await locks.releaseByTaskId(taskId);
@@ -249,7 +249,7 @@ async function notifyBoxUpdate() {
if (!version) {
await eventlog.add(eventlog.ACTION_INSTALL_FINISH, AuditSource.CRON, { version: constants.VERSION });
} else {
debug(`notifyBoxUpdate: update finished from ${version} to ${constants.VERSION}`);
log(`notifyBoxUpdate: update finished from ${version} to ${constants.VERSION}`);
const [error] = await safe(tasks.setCompletedByType(tasks.TASK_BOX_UPDATE, { error: null }));
if (error && error.reason !== BoxError.NOT_FOUND) throw error; // when hotfixing, task may not exist
await eventlog.add(eventlog.ACTION_UPDATE_FINISH, AuditSource.CRON, { errorMessage: '', oldVersion: version || 'dev', newVersion: constants.VERSION });
@@ -265,10 +265,10 @@ async function autoUpdate(auditSource) {
const boxUpdateInfo = await getBoxUpdate();
// do box before app updates. for the off chance that the box logic fixes some app update logic issue
if (boxUpdateInfo && !boxUpdateInfo.unstable) {
debug('autoUpdate: starting box autoupdate to %j', boxUpdateInfo.version);
log('autoUpdate: starting box autoupdate to %j', boxUpdateInfo.version);
const [error] = await safe(startBoxUpdateTask({ skipBackup: false }, AuditSource.CRON));
if (!error) return; // do not start app updates when a box update got scheduled
debug(`autoUpdate: failed to start box autoupdate task: ${error.message}`);
log(`autoUpdate: failed to start box autoupdate task: ${error.message}`);
// fall through to update apps if box update never started (failed ubuntu or avx check)
}
@@ -276,13 +276,13 @@ async function autoUpdate(auditSource) {
for (const app of result) {
if (!app.updateInfo) continue;
if (!app.updateInfo.isAutoUpdatable) {
debug(`autoUpdate: ${app.fqdn} requires manual update. skipping`);
log(`autoUpdate: ${app.fqdn} requires manual update. skipping`);
continue;
}
const sites = await backupSites.listByContentForUpdates(app.id);
if (sites.length === 0) {
debug(`autoUpdate: ${app.fqdn} has no backup site for updates. skipping`);
log(`autoUpdate: ${app.fqdn} has no backup site for updates. skipping`);
continue;
}
@@ -291,9 +291,9 @@ async function autoUpdate(auditSource) {
force: false
};
debug(`autoUpdate: ${app.fqdn} will be automatically updated`);
log(`autoUpdate: ${app.fqdn} will be automatically updated`);
const [updateError] = await safe(apps.updateApp(app, data, auditSource));
if (updateError) debug(`autoUpdate: error autoupdating ${app.fqdn}: ${updateError.message}`);
if (updateError) log(`autoUpdate: error autoupdating ${app.fqdn}: ${updateError.message}`);
}
}
@@ -320,7 +320,7 @@ async function checkAppUpdate(app, options) {
async function checkBoxUpdate(options) {
assert.strictEqual(typeof options, 'object');
debug('checkBoxUpdate: checking for updates');
log('checkBoxUpdate: checking for updates');
const updateInfo = await appstore.getBoxUpdate(options);
if (updateInfo) {
@@ -346,7 +346,7 @@ async function raiseNotifications() {
// currently, we do not raise notifications when auto-update is disabled. separate notifications appears spammy when having many apps
// in the future, we can maybe aggregate?
if (app.updateInfo && !app.updateInfo.isAutoUpdatable) {
debug(`autoUpdate: ${app.fqdn} cannot be autoupdated. skipping`);
log(`autoUpdate: ${app.fqdn} cannot be autoupdated. skipping`);
await notifications.pin(notifications.TYPE_MANUAL_APP_UPDATE_NEEDED, `${app.manifest.title} at ${app.fqdn} requires manual update to version ${app.updateInfo.manifest.version}`,
`Changelog:\n${app.updateInfo.manifest.changelog}\n`, { context: app.id });
continue;
@@ -358,12 +358,12 @@ async function checkForUpdates(options) {
assert.strictEqual(typeof options, 'object');
const [boxError] = await safe(checkBoxUpdate(options));
if (boxError) debug('checkForUpdates: error checking for box updates: %o', boxError);
if (boxError) log('checkForUpdates: error checking for box updates: %o', boxError);
// check app updates
const result = await apps.list();
for (const app of result) {
await safe(checkAppUpdate(app, options), { debug });
await safe(checkAppUpdate(app, options), { debug: log });
}
// raise notifications here because the updatechecker runs regardless of auto-updater cron job

Some files were not shown because too many files have changed in this diff Show More