From 01d0c738bc32bd0fd3dede81023cc8d6b2d49214 Mon Sep 17 00:00:00 2001 From: Girish Ramakrishnan Date: Thu, 12 Mar 2026 22:55:28 +0530 Subject: [PATCH] replace debug() with our custom logger mostly we want trace() and log(). trace() can be enabled whenever we want by flipping a flag and restarting box --- box.js | 14 +- package-lock.json | 1 - package.json | 1 - setup/start/systemd/box.service | 3 +- setup/start/systemd/cloudron-syslog.service | 2 +- src/acme2.js | 84 ++++---- src/apphealthmonitor.js | 20 +- src/applinks.js | 20 +- src/apps.js | 60 +++--- src/appstore.js | 24 +-- src/apptask.js | 34 ++-- src/apptaskmanager.js | 12 +- src/asynctask.js | 10 +- src/backupcleaner.js | 44 ++-- src/backupformat/rsync.js | 40 ++-- src/backupformat/tgz.js | 56 +++--- src/backupintegrity.js | 6 +- src/backups.js | 8 +- src/backupsites.js | 20 +- src/backuptask.js | 64 +++--- src/branding.js | 6 +- src/community.js | 8 +- src/cron.js | 54 ++--- src/dashboard.js | 18 +- src/database.js | 14 +- src/df.js | 8 +- src/directoryserver.js | 34 ++-- src/dns.js | 28 +-- src/dns/bunny.js | 20 +- src/dns/cloudflare.js | 24 +-- src/dns/desec.js | 10 +- src/dns/digitalocean.js | 18 +- src/dns/dnsimple.js | 20 +- src/dns/gandi.js | 16 +- src/dns/gcdns.js | 12 +- src/dns/godaddy.js | 16 +- src/dns/hetzner.js | 18 +- src/dns/hetznercloud.js | 16 +- src/dns/inwx.js | 18 +- src/dns/linode.js | 18 +- src/dns/manual.js | 6 +- src/dns/namecheap.js | 14 +- src/dns/namecom.js | 20 +- src/dns/netcup.js | 18 +- src/dns/noop.js | 6 +- src/dns/ovh.js | 20 +- src/dns/porkbun.js | 18 +- src/dns/route53.js | 14 +- src/dns/vultr.js | 18 +- src/dns/waitfordns.js | 28 +-- src/dns/wildcard.js | 6 +- src/docker.js | 46 ++--- src/dockerproxy.js | 16 +- src/domains.js | 10 +- src/dyndns.js | 20 +- src/eventlog.js | 6 +- src/externalldap.js | 78 ++++---- src/hush.js | 6 +- src/janitor.js | 18 +- src/ldapserver.js | 46 ++--- src/locks.js | 24 +-- src/logger.js | 16 ++ src/logs.js | 6 +- src/mail.js | 36 ++-- src/mailer.js | 10 +- src/mailserver.js | 28 +-- src/metrics.js | 12 +- src/mounts.js | 6 +- src/network/generic.js | 10 +- src/network/network-interface.js | 8 +- src/notifications.js | 22 +- src/oidcserver.js | 52 ++--- src/once.js | 6 +- src/openssl.js | 14 +- src/passkeys.js | 20 +- src/platform.js | 64 +++--- src/provision.js | 30 +-- src/proxyauth.js | 10 +- src/reverseproxy.js | 58 +++--- src/routes/accesscontrol.js | 6 +- src/routes/apps.js | 8 +- src/routes/auth.js | 6 +- src/routes/mailserver.js | 6 +- src/routes/test/common.js | 26 +-- src/scheduler.js | 32 +-- src/scripts/backupupload.js | 10 +- src/scripts/starttask.sh | 2 +- src/server.js | 12 +- src/services.js | 210 ++++++++++---------- src/sftp.js | 16 +- src/shell.js | 20 +- src/storage/filesystem.js | 18 +- src/storage/gcs.js | 18 +- src/storage/s3.js | 20 +- src/syncer.js | 14 +- src/system.js | 16 +- src/tasks.js | 28 +-- src/taskworker.js | 16 +- src/translations.js | 10 +- src/updater.js | 48 ++--- src/user-directory.js | 6 +- src/users.js | 34 ++-- src/volumes.js | 12 +- syslog.js | 22 +- 104 files changed, 1187 insertions(+), 1174 deletions(-) create mode 100644 src/logger.js diff --git a/box.js b/box.js index b7cea83fb..6298b324c 100755 --- a/box.js +++ b/box.js @@ -10,9 +10,9 @@ import proxyAuth from './src/proxyauth.js'; import safe from 'safetydance'; import server from './src/server.js'; import directoryServer from './src/directoryserver.js'; -import debugModule from 'debug'; +import logger from './src/logger.js'; -const debug = debugModule('box:box'); +const { log, trace } = logger('box'); let logFd; @@ -59,13 +59,13 @@ const [error] = await safe(startServers()); if (error) exitSync({ error, code: 1, message: 'Error starting servers' }); process.on('SIGHUP', async function () { - debug('Received SIGHUP. Re-reading configs.'); + log('Received SIGHUP. Re-reading configs.'); const conf = await directoryServer.getConfig(); if (conf.enabled) await directoryServer.checkCertificate(); }); process.on('SIGINT', async function () { - debug('Received SIGINT. Shutting down.'); + log('Received SIGINT. Shutting down.'); await proxyAuth.stop(); await server.stop(); @@ -74,13 +74,13 @@ process.on('SIGINT', async function () { await oidcServer.stop(); setTimeout(() => { - debug('Shutdown complete'); + log('Shutdown complete'); process.exit(); }, 2000); // need to wait for the task processes to die }); process.on('SIGTERM', async function () { - debug('Received SIGTERM. Shutting down.'); + log('Received SIGTERM. Shutting down.'); await proxyAuth.stop(); await server.stop(); @@ -89,7 +89,7 @@ process.on('SIGTERM', async function () { await oidcServer.stop(); setTimeout(() => { - debug('Shutdown complete'); + log('Shutdown complete'); process.exit(); }, 2000); // need to wait for the task processes to die }); diff --git a/package-lock.json b/package-lock.json index ed2b587cc..cb3a82e15 100644 --- a/package-lock.json +++ b/package-lock.json @@ -29,7 +29,6 @@ "cron": "^4.4.0", "db-migrate": "^0.11.14", "db-migrate-mysql": "^3.0.0", - "debug": "^4.4.3", "dockerode": "^4.0.9", "domrobot-client": "^3.3.0", "ejs": "^4.0.1", diff --git a/package.json b/package.json index 71b1b22b9..01d5f4d98 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,6 @@ "cron": "^4.4.0", "db-migrate": "^0.11.14", "db-migrate-mysql": "^3.0.0", - "debug": "^4.4.3", "dockerode": "^4.0.9", "domrobot-client": "^3.3.0", "ejs": "^4.0.1", diff --git a/setup/start/systemd/box.service b/setup/start/systemd/box.service index b355e6a67..8c5606967 100644 --- a/setup/start/systemd/box.service +++ b/setup/start/systemd/box.service @@ -16,8 +16,7 @@ Restart=always ExecStart=/home/yellowtent/box/box.js ExecReload=/bin/kill -HUP $MAINPID ; we run commands like df which will parse properly only with correct locale -; add "oidc-provider:*" to DEBUG for OpenID debugging -Environment="HOME=/home/yellowtent" "USER=yellowtent" "DEBUG=box:*,-box:ldapserver,-box:directoryserver,-box:oidcserver" "BOX_ENV=cloudron" "NODE_ENV=production" "LC_ALL=C" +Environment="HOME=/home/yellowtent" "USER=yellowtent" "BOX_ENV=cloudron" "NODE_ENV=production" "LC_ALL=C" ; this sends the main process SIGTERM and then if anything lingers to the control-group . this is also the case if the main process crashes. ; the box code handles SIGTERM and cleanups the tasks KillMode=mixed diff --git a/setup/start/systemd/cloudron-syslog.service b/setup/start/systemd/cloudron-syslog.service index 9dcc35f0d..7127f0730 100644 --- a/setup/start/systemd/cloudron-syslog.service +++ b/setup/start/systemd/cloudron-syslog.service @@ -5,7 +5,7 @@ After=network.target [Service] ExecStart=/home/yellowtent/box/syslog.js WorkingDirectory=/home/yellowtent/box -Environment="NODE_ENV=production" "DEBUG=syslog:*" "BOX_ENV=cloudron" +Environment="NODE_ENV=production" "BOX_ENV=cloudron" Restart=always User=yellowtent Group=yellowtent diff --git a/src/acme2.js b/src/acme2.js index d0bcd87ff..754d96983 100644 --- a/src/acme2.js +++ b/src/acme2.js @@ -2,7 +2,7 @@ import assert from 'node:assert'; import blobs from './blobs.js'; import BoxError from './boxerror.js'; import crypto from 'node:crypto'; -import debugModule from 'debug'; +import logger from './logger.js'; import dns from './dns.js'; import openssl from './openssl.js'; import path from 'node:path'; @@ -12,7 +12,7 @@ import safe from 'safetydance'; import superagent from '@cloudron/superagent'; import users from './users.js'; -const debug = debugModule('box:cert/acme2'); +const { log, trace } = logger('cert/acme2'); const CA_PROD_DIRECTORY_URL = 'https://acme-v02.api.letsencrypt.org/directory', CA_STAGING_DIRECTORY_URL = 'https://acme-staging-v02.api.letsencrypt.org/directory'; @@ -53,7 +53,7 @@ function Acme2(fqdn, domainObject, email, key, options) { this.profile = options.profile || ''; // https://letsencrypt.org/docs/profiles/ . is validated against the directory - debug(`Acme2: will get cert for fqdn: ${this.fqdn} cn: ${this.cn} certName: ${this.certName} wildcard: ${this.wildcard} http: ${this.forceHttpAuthorization}`); + log(`Acme2: will get cert for fqdn: ${this.fqdn} cn: ${this.cn} certName: ${this.certName} wildcard: ${this.wildcard} http: ${this.forceHttpAuthorization}`); } // urlsafe base64 encoding (jose) @@ -140,7 +140,7 @@ Acme2.prototype.sendSignedRequest = async function (url, payload) { const nonce = response.headers['Replay-Nonce'.toLowerCase()]; if (!nonce) throw new BoxError(BoxError.ACME_ERROR, 'No nonce in response'); - debug(`sendSignedRequest: using nonce ${nonce} for url ${url}`); + log(`sendSignedRequest: using nonce ${nonce} for url ${url}`); const protected64 = b64(JSON.stringify(Object.assign({}, header, { nonce: nonce }))); @@ -168,7 +168,7 @@ Acme2.prototype.postAsGet = async function (url) { Acme2.prototype.updateContact = async function (registrationUri) { assert.strictEqual(typeof registrationUri, 'string'); - debug(`updateContact: registrationUri: ${registrationUri} email: ${this.email}`); + log(`updateContact: registrationUri: ${registrationUri} email: ${this.email}`); // https://github.com/ietf-wg-acme/acme/issues/30 const payload = { @@ -178,7 +178,7 @@ Acme2.prototype.updateContact = async function (registrationUri) { const result = await this.sendSignedRequest(registrationUri, JSON.stringify(payload)); if (result.status !== 200) throw new BoxError(BoxError.ACME_ERROR, `Failed to update contact. Expecting 200, got ${result.status} ${result.text}`); - debug(`updateContact: contact of user updated to ${this.email}`); + log(`updateContact: contact of user updated to ${this.email}`); }; Acme2.prototype.ensureAccount = async function () { @@ -186,18 +186,18 @@ Acme2.prototype.ensureAccount = async function () { termsOfServiceAgreed: true }; - debug('ensureAccount: registering user'); + log('ensureAccount: registering user'); this.accountKey = await blobs.getString(blobs.ACME_ACCOUNT_KEY); if (!this.accountKey) { - debug('ensureAccount: generating new account keys'); + log('ensureAccount: generating new account keys'); this.accountKey = await openssl.generateKey('rsa4096'); await blobs.setString(blobs.ACME_ACCOUNT_KEY, this.accountKey); } let result = await this.sendSignedRequest(this.directory.newAccount, JSON.stringify(payload)); if (result.status === 403 && result.body.type === 'urn:ietf:params:acme:error:unauthorized') { - debug(`ensureAccount: key was revoked. ${result.status} ${result.text}. generating new account key`); + log(`ensureAccount: key was revoked. ${result.status} ${result.text}. generating new account key`); this.accountKey = await openssl.generateKey('rsa4096'); await blobs.setString(blobs.ACME_ACCOUNT_KEY, this.accountKey); result = await this.sendSignedRequest(this.directory.newAccount, JSON.stringify(payload)); @@ -206,7 +206,7 @@ Acme2.prototype.ensureAccount = async function () { // 200 if already exists. 201 for new accounts if (result.status !== 200 && result.status !== 201) throw new BoxError(BoxError.ACME_ERROR, `Failed to register new account. Expecting 200 or 201, got ${result.status} ${result.text}`); - debug(`ensureAccount: user registered keyid: ${result.headers.location}`); + log(`ensureAccount: user registered keyid: ${result.headers.location}`); this.accountKeyId = result.headers.location; @@ -223,14 +223,14 @@ Acme2.prototype.newOrder = async function () { }); }); - debug(`newOrder: ${JSON.stringify(this.altNames)}`); + log(`newOrder: ${JSON.stringify(this.altNames)}`); const result = await this.sendSignedRequest(this.directory.newOrder, JSON.stringify(payload)); if (result.status === 403) throw new BoxError(BoxError.ACCESS_DENIED, `Forbidden sending new order: ${result.body.detail}`); if (result.status !== 201) throw new BoxError(BoxError.ACME_ERROR, `Failed to send new order. Expecting 201, got ${result.status} ${result.text}`); const order = result.body, orderUrl = result.headers.location; - debug(`newOrder: created order ${this.cn} order: ${result.text} orderUrl: ${orderUrl}`); + log(`newOrder: created order ${this.cn} order: ${result.text} orderUrl: ${orderUrl}`); if (!Array.isArray(order.authorizations)) throw new BoxError(BoxError.ACME_ERROR, 'invalid authorizations in order'); if (typeof order.finalize !== 'string') throw new BoxError(BoxError.ACME_ERROR, 'invalid finalize in order'); @@ -242,18 +242,18 @@ Acme2.prototype.newOrder = async function () { Acme2.prototype.waitForOrder = async function (orderUrl) { assert.strictEqual(typeof orderUrl, 'string'); - debug(`waitForOrder: ${orderUrl}`); + log(`waitForOrder: ${orderUrl}`); - return await promiseRetry({ times: 15, interval: 20000, debug }, async () => { - debug('waitForOrder: getting status'); + return await promiseRetry({ times: 15, interval: 20000, debug: log }, async () => { + log('waitForOrder: getting status'); const result = await this.postAsGet(orderUrl); if (result.status !== 200) { - debug(`waitForOrder: invalid response code getting uri ${result.status}`); + log(`waitForOrder: invalid response code getting uri ${result.status}`); throw new BoxError(BoxError.ACME_ERROR, `Bad response when waiting for order. code: ${result.status}`); } - debug('waitForOrder: status is "%s %j', result.body.status, result.body); + log('waitForOrder: status is "%s %j', result.body.status, result.body); if (result.body.status === 'pending' || result.body.status === 'processing') throw new BoxError(BoxError.ACME_ERROR, `Request is in ${result.body.status} state`); else if (result.body.status === 'valid' && result.body.certificate) return result.body.certificate; @@ -279,7 +279,7 @@ Acme2.prototype.getKeyAuthorization = async function (token) { Acme2.prototype.notifyChallengeReady = async function (challenge) { assert.strictEqual(typeof challenge, 'object'); // { type, status, url, token } - debug(`notifyChallengeReady: ${challenge.url} was met`); + log(`notifyChallengeReady: ${challenge.url} was met`); const keyAuthorization = await this.getKeyAuthorization(challenge.token); @@ -295,18 +295,18 @@ Acme2.prototype.notifyChallengeReady = async function (challenge) { Acme2.prototype.waitForChallenge = async function (challenge) { assert.strictEqual(typeof challenge, 'object'); - debug(`waitingForChallenge: ${JSON.stringify(challenge)}`); + log(`waitingForChallenge: ${JSON.stringify(challenge)}`); - await promiseRetry({ times: 15, interval: 20000, debug }, async () => { - debug('waitingForChallenge: getting status'); + await promiseRetry({ times: 15, interval: 20000, debug: log }, async () => { + log('waitingForChallenge: getting status'); const result = await this.postAsGet(challenge.url); if (result.status !== 200) { - debug(`waitForChallenge: invalid response code getting uri ${result.status}`); + log(`waitForChallenge: invalid response code getting uri ${result.status}`); throw new BoxError(BoxError.ACME_ERROR, `Bad response code when waiting for challenge : ${result.status}`); } - debug(`waitForChallenge: status is "${result.body.status}" "${result.text}"`); + log(`waitForChallenge: status is "${result.body.status}" "${result.text}"`); if (result.body.status === 'pending') throw new BoxError(BoxError.ACME_ERROR, 'Challenge is in pending state'); else if (result.body.status === 'valid') return; @@ -325,7 +325,7 @@ Acme2.prototype.signCertificate = async function (finalizationUrl, csrPem) { csr: b64(csrDer) }; - debug(`signCertificate: sending sign request to ${finalizationUrl}`); + log(`signCertificate: sending sign request to ${finalizationUrl}`); const result = await this.sendSignedRequest(finalizationUrl, JSON.stringify(payload)); // 429 means we reached the cert limit for this domain @@ -335,8 +335,8 @@ Acme2.prototype.signCertificate = async function (finalizationUrl, csrPem) { Acme2.prototype.downloadCertificate = async function (certUrl) { assert.strictEqual(typeof certUrl, 'string'); - return await promiseRetry({ times: 5, interval: 20000, debug }, async () => { - debug(`downloadCertificate: downloading certificate of ${this.cn}`); + return await promiseRetry({ times: 5, interval: 20000, debug: log }, async () => { + log(`downloadCertificate: downloading certificate of ${this.cn}`); const result = await this.postAsGet(certUrl); if (result.status === 202) throw new BoxError(BoxError.ACME_ERROR, 'Retry downloading certificate'); @@ -350,12 +350,12 @@ Acme2.prototype.downloadCertificate = async function (certUrl) { Acme2.prototype.prepareHttpChallenge = async function (challenge) { assert.strictEqual(typeof challenge, 'object'); - debug(`prepareHttpChallenge: preparing for challenge ${JSON.stringify(challenge)}`); + log(`prepareHttpChallenge: preparing for challenge ${JSON.stringify(challenge)}`); const keyAuthorization = await this.getKeyAuthorization(challenge.token); const challengeFilePath = path.join(paths.ACME_CHALLENGES_DIR, challenge.token); - debug(`prepareHttpChallenge: writing ${keyAuthorization} to ${challengeFilePath}`); + log(`prepareHttpChallenge: writing ${keyAuthorization} to ${challengeFilePath}`); if (!safe.fs.writeFileSync(challengeFilePath, keyAuthorization)) throw new BoxError(BoxError.FS_ERROR, `Error writing challenge: ${safe.error.message}`); }; @@ -364,7 +364,7 @@ Acme2.prototype.cleanupHttpChallenge = async function (challenge) { assert.strictEqual(typeof challenge, 'object'); const challengeFilePath = path.join(paths.ACME_CHALLENGES_DIR, challenge.token); - debug(`cleanupHttpChallenge: unlinking ${challengeFilePath}`); + log(`cleanupHttpChallenge: unlinking ${challengeFilePath}`); if (!safe.fs.unlinkSync(challengeFilePath)) throw new BoxError(BoxError.FS_ERROR, `Error unlinking challenge: ${safe.error.message}`); }; @@ -381,7 +381,7 @@ function getChallengeSubdomain(cn, domain) { challengeSubdomain = '_acme-challenge.' + cn.slice(0, -domain.length - 1); } - debug(`getChallengeSubdomain: challenge subdomain for cn ${cn} at domain ${domain} is ${challengeSubdomain}`); + log(`getChallengeSubdomain: challenge subdomain for cn ${cn} at domain ${domain} is ${challengeSubdomain}`); return challengeSubdomain; } @@ -389,7 +389,7 @@ function getChallengeSubdomain(cn, domain) { Acme2.prototype.prepareDnsChallenge = async function (cn, challenge) { assert.strictEqual(typeof challenge, 'object'); - debug(`prepareDnsChallenge: preparing for challenge: ${JSON.stringify(challenge)}`); + log(`prepareDnsChallenge: preparing for challenge: ${JSON.stringify(challenge)}`); const keyAuthorization = await this.getKeyAuthorization(challenge.token); const shasum = crypto.createHash('sha256'); @@ -398,7 +398,7 @@ Acme2.prototype.prepareDnsChallenge = async function (cn, challenge) { const txtValue = urlBase64Encode(shasum.digest('base64')); const challengeSubdomain = getChallengeSubdomain(cn, this.domain); - debug(`prepareDnsChallenge: update ${challengeSubdomain} with ${txtValue}`); + log(`prepareDnsChallenge: update ${challengeSubdomain} with ${txtValue}`); await dns.upsertDnsRecords(challengeSubdomain, this.domain, 'TXT', [ `"${txtValue}"` ]); @@ -416,7 +416,7 @@ Acme2.prototype.cleanupDnsChallenge = async function (cn, challenge) { const txtValue = urlBase64Encode(shasum.digest('base64')); const challengeSubdomain = getChallengeSubdomain(cn, this.domain); - debug(`cleanupDnsChallenge: remove ${challengeSubdomain} with ${txtValue}`); + log(`cleanupDnsChallenge: remove ${challengeSubdomain} with ${txtValue}`); await dns.removeDnsRecords(challengeSubdomain, this.domain, 'TXT', [ `"${txtValue}"` ]); }; @@ -425,7 +425,7 @@ Acme2.prototype.prepareChallenge = async function (cn, authorization) { assert.strictEqual(typeof cn, 'string'); assert.strictEqual(typeof authorization, 'object'); - debug(`prepareChallenge: http: ${this.forceHttpAuthorization} cn: ${cn} authorization: ${JSON.stringify(authorization)}`); + log(`prepareChallenge: http: ${this.forceHttpAuthorization} cn: ${cn} authorization: ${JSON.stringify(authorization)}`); // validation is cached by LE for 60 days or so. if a user switches from non-wildcard DNS (http challenge) to programmatic DNS (dns challenge), then // LE remembers the challenge type and won't give us a dns challenge for 60 days! @@ -447,7 +447,7 @@ Acme2.prototype.cleanupChallenge = async function (cn, challenge) { assert.strictEqual(typeof cn, 'string'); assert.strictEqual(typeof challenge, 'object'); - debug(`cleanupChallenge: http: ${this.forceHttpAuthorization}`); + log(`cleanupChallenge: http: ${this.forceHttpAuthorization}`); if (this.forceHttpAuthorization) { await this.cleanupHttpChallenge(challenge); @@ -461,7 +461,7 @@ Acme2.prototype.acmeFlow = async function () { const { order, orderUrl } = await this.newOrder(); for (const authorizationUrl of order.authorizations) { - debug(`acmeFlow: authorizing ${authorizationUrl}`); + log(`acmeFlow: authorizing ${authorizationUrl}`); const response = await this.postAsGet(authorizationUrl); if (response.status !== 200) throw new BoxError(BoxError.ACME_ERROR, `Invalid response code getting authorization : ${response.status}`); @@ -471,7 +471,7 @@ Acme2.prototype.acmeFlow = async function () { const challenge = await this.prepareChallenge(cn, authorization); await this.notifyChallengeReady(challenge); await this.waitForChallenge(challenge); - await safe(this.cleanupChallenge(cn, challenge), { debug }); + await safe(this.cleanupChallenge(cn, challenge), { debug: log }); } const csr = await openssl.createCsr(this.key, this.cn, this.altNames); @@ -485,7 +485,7 @@ Acme2.prototype.acmeFlow = async function () { }; Acme2.prototype.loadDirectory = async function () { - await promiseRetry({ times: 3, interval: 20000, debug }, async () => { + await promiseRetry({ times: 3, interval: 20000, debug: log }, async () => { const response = await superagent.get(this.caDirectory).timeout(30000).ok(() => true); if (response.status !== 200) throw new BoxError(BoxError.ACME_ERROR, `Invalid response code when fetching directory : ${response.status}`); @@ -502,11 +502,11 @@ Acme2.prototype.loadDirectory = async function () { }; Acme2.prototype.getCertificate = async function () { - debug(`getCertificate: start acme flow for ${this.cn} from ${this.caDirectory}`); + log(`getCertificate: start acme flow for ${this.cn} from ${this.caDirectory}`); await this.loadDirectory(); const result = await this.acmeFlow(); // { key, cert, csr, renewalInfo } - debug(`getCertificate: acme flow completed for ${this.cn}. renewalInfo: ${JSON.stringify(result.renewalInfo)}`); + log(`getCertificate: acme flow completed for ${this.cn}. renewalInfo: ${JSON.stringify(result.renewalInfo)}`); return result; }; @@ -522,8 +522,8 @@ async function getCertificate(fqdn, domainObject, key) { const owner = await users.getOwner(); const email = owner?.email || 'webmaster@cloudron.io'; // can error if not activated yet - return await promiseRetry({ times: 3, interval: 0, debug }, async function () { - debug(`getCertificate: for fqdn ${fqdn} and domain ${domainObject.domain}`); + return await promiseRetry({ times: 3, interval: 0, debug: log }, async function () { + log(`getCertificate: for fqdn ${fqdn} and domain ${domainObject.domain}`); const acme = new Acme2(fqdn, domainObject, email, key, { /* profile: 'shortlived' */ }); return await acme.getCertificate(); diff --git a/src/apphealthmonitor.js b/src/apphealthmonitor.js index 1c497db31..cc0ae64ec 100644 --- a/src/apphealthmonitor.js +++ b/src/apphealthmonitor.js @@ -3,13 +3,13 @@ import assert from 'node:assert'; import AuditSource from './auditsource.js'; import BoxError from './boxerror.js'; import constants from './constants.js'; -import debugModule from 'debug'; +import logger from './logger.js'; import docker from './docker.js'; import eventlog from './eventlog.js'; import safe from 'safetydance'; import superagent from '@cloudron/superagent'; -const debug = debugModule('box:apphealthmonitor'); +const { log, trace } = logger('apphealthmonitor'); const UNHEALTHY_THRESHOLD = 20 * 60 * 1000; // 20 minutes @@ -32,20 +32,20 @@ async function setHealth(app, health) { if (health === apps.HEALTH_HEALTHY) { healthTime = now; if (lastHealth && lastHealth !== apps.HEALTH_HEALTHY) { // app starts out with null health - debug(`setHealth: ${app.id} (${app.fqdn}) switched from ${lastHealth} to healthy`); + log(`setHealth: ${app.id} (${app.fqdn}) switched from ${lastHealth} to healthy`); // do not send mails for dev apps if (!app.debugMode) await eventlog.add(eventlog.ACTION_APP_UP, AuditSource.HEALTH_MONITOR, { app }); } } else if (Math.abs(now - healthTime) > UNHEALTHY_THRESHOLD) { if (lastHealth === apps.HEALTH_HEALTHY) { - debug(`setHealth: marking ${app.id} (${app.fqdn}) as unhealthy since not seen for more than ${UNHEALTHY_THRESHOLD/(60 * 1000)} minutes`); + log(`setHealth: marking ${app.id} (${app.fqdn}) as unhealthy since not seen for more than ${UNHEALTHY_THRESHOLD/(60 * 1000)} minutes`); // do not send mails for dev apps if (!app.debugMode) await eventlog.add(eventlog.ACTION_APP_DOWN, AuditSource.HEALTH_MONITOR, { app }); } } else { - debug(`setHealth: ${app.id} (${app.fqdn}) waiting for ${(UNHEALTHY_THRESHOLD - Math.abs(now - healthTime))/1000} to update health`); + log(`setHealth: ${app.id} (${app.fqdn}) waiting for ${(UNHEALTHY_THRESHOLD - Math.abs(now - healthTime))/1000} to update health`); return; } @@ -137,7 +137,7 @@ async function processDockerEvents(options) { const now = Date.now(); const notifyUser = !info?.app?.debugMode && ((now - gLastOomMailTime) > OOM_EVENT_LIMIT); - debug(`OOM ${program} notifyUser: ${notifyUser}. lastOomTime: ${gLastOomMailTime} (now: ${now})`); + log(`OOM ${program} notifyUser: ${notifyUser}. lastOomTime: ${gLastOomMailTime} (now: ${now})`); if (notifyUser) { await eventlog.add(eventlog.ACTION_APP_OOM, AuditSource.HEALTH_MONITOR, { event, containerId, addonName: info?.addonName || null, app: info?.app || null }); @@ -147,11 +147,11 @@ async function processDockerEvents(options) { }); stream.on('error', function (error) { - debug('Error reading docker events: %o', error); + log('Error reading docker events: %o', error); }); stream.on('end', function () { - // debug('Event stream ended'); + // log('Event stream ended'); }); // safety hatch if 'until' doesn't work (there are cases where docker is working with a different time) @@ -167,12 +167,12 @@ async function processApp(options) { const results = await Promise.allSettled(healthChecks); // wait for all promises to finish const unfulfilled = results.filter(r => r.status === 'rejected'); - if (unfulfilled.length) debug(`app health: ${unfulfilled.length} health checks exceptions. e.g. ${unfulfilled[0].reason}`); // this should not happen + if (unfulfilled.length) log(`app health: ${unfulfilled.length} health checks exceptions. e.g. ${unfulfilled[0].reason}`); // this should not happen const stopped = allApps.filter(app => app.runState === apps.RSTATE_STOPPED); const running = allApps.filter(function (a) { return a.installationState === apps.ISTATE_INSTALLED && a.runState === apps.RSTATE_RUNNING && a.health === apps.HEALTH_HEALTHY; }); - debug(`app health: ${running.length} running / ${stopped.length} stopped / ${allApps.length - running.length - stopped.length} unresponsive`); + log(`app health: ${running.length} running / ${stopped.length} stopped / ${allApps.length - running.length - stopped.length} unresponsive`); } async function run(intervalSecs) { diff --git a/src/applinks.js b/src/applinks.js index 3a2b81e76..564498282 100644 --- a/src/applinks.js +++ b/src/applinks.js @@ -3,12 +3,12 @@ import apps from './apps.js'; import BoxError from './boxerror.js'; import crypto from 'node:crypto'; import database from './database.js'; -import debugModule from 'debug'; +import logger from './logger.js'; import jsdom from 'jsdom'; import safe from 'safetydance'; import superagent from '@cloudron/superagent'; -const debug = debugModule('box:applinks'); +const { log, trace } = logger('applinks'); const APPLINKS_FIELDS= [ 'id', 'accessRestrictionJson', 'creationTime', 'updateTime', 'ts', 'label', 'tagsJson', 'icon', 'upstreamUri' ].join(','); @@ -84,7 +84,7 @@ async function detectMetaInfo(upstreamUri) { const [upstreamError, upstreamRespose] = await safe(superagent.get(upstreamUri).set('User-Agent', 'Mozilla').timeout(10*1000)); if (upstreamError) { - debug(`detectMetaInfo: error fetching ${upstreamUri}: ${upstreamError.status}`); + log(`detectMetaInfo: error fetching ${upstreamUri}: ${upstreamError.status}`); return null; } @@ -125,21 +125,21 @@ async function detectMetaInfo(upstreamUri) { if (favicon) { favicon = new URL(favicon, upstreamRespose.url).toString(); - debug(`detectMetaInfo: found icon: ${favicon}`); + log(`detectMetaInfo: found icon: ${favicon}`); const [error, response] = await safe(superagent.get(favicon).timeout(10*1000)); - if (error) debug(`Failed to fetch icon ${favicon}: ${error.message} ${error.status}`); + if (error) log(`Failed to fetch icon ${favicon}: ${error.message} ${error.status}`); else if (response.headers['content-type']?.indexOf('image') !== -1) icon = response.body || response.text; - else debug(`Failed to fetch icon ${favicon}: status=${response.status}`); + else log(`Failed to fetch icon ${favicon}: status=${response.status}`); } if (!favicon) { - debug(`Unable to find a suitable icon for ${upstreamUri}, try fallback favicon.ico`); + log(`Unable to find a suitable icon for ${upstreamUri}, try fallback favicon.ico`); const [error, response] = await safe(superagent.get(`${upstreamUri}/favicon.ico`).timeout(10*1000)); - if (error) debug(`Failed to fetch icon ${favicon}: ${error.message}`); + if (error) log(`Failed to fetch icon ${favicon}: ${error.message}`); else if (response.headers['content-type']?.indexOf('image') !== -1) icon = response.body || response.text; - else debug(`Failed to fetch icon ${favicon}: status=${response.status} content type ${response.headers['content-type']}`); + else log(`Failed to fetch icon ${favicon}: status=${response.status} content type ${response.headers['content-type']}`); } // detect label @@ -153,7 +153,7 @@ async function detectMetaInfo(upstreamUri) { async function add(applink) { assert.strictEqual(typeof applink, 'object'); - debug(`add: ${applink.upstreamUri}`); + log(`add: ${applink.upstreamUri}`); let error = validateUpstreamUri(applink.upstreamUri); if (error) throw error; diff --git a/src/apps.js b/src/apps.js index f4865c490..1d150a443 100644 --- a/src/apps.js +++ b/src/apps.js @@ -10,7 +10,7 @@ import crypto from 'node:crypto'; import { CronTime } from 'cron'; import dashboard from './dashboard.js'; import database from './database.js'; -import debugModule from 'debug'; +import logger from './logger.js'; import dns from './dns.js'; import docker from './docker.js'; import domains from './domains.js'; @@ -40,7 +40,7 @@ import util from 'node:util'; import volumes from './volumes.js'; import _ from './underscore.js'; -const debug = debugModule('box:apps'); +const { log, trace } = logger('apps'); const shell = shellModule('apps'); const PORT_TYPE_TCP = 'tcp'; @@ -306,7 +306,7 @@ function getDuplicateErrorDetails(errorMessage, locations, portBindings) { const match = errorMessage.match(/Duplicate entry '(.*)' for key '(.*)'/); if (!match) { - debug('Unexpected SQL error message.', errorMessage); + log('Unexpected SQL error message.', errorMessage); return new BoxError(BoxError.DATABASE_ERROR, errorMessage); } @@ -1058,7 +1058,7 @@ async function onTaskFinished(error, appId, installationState, taskId, auditSour switch (installationState) { case ISTATE_PENDING_DATA_DIR_MIGRATION: - if (success) await safe(services.rebuildService('sftp', auditSource), { debug }); + if (success) await safe(services.rebuildService('sftp', auditSource), { debug: log }); break; case ISTATE_PENDING_UPDATE: { const fromManifest = success ? task.args[1].updateConfig.manifest : app.manifest; @@ -1071,8 +1071,8 @@ async function onTaskFinished(error, appId, installationState, taskId, auditSour } // this can race with an new install task but very unlikely - debug(`onTaskFinished: checking to stop unused services. hasPending: ${appTaskManager.hasPendingTasks()}`) - if (!appTaskManager.hasPendingTasks()) safe(services.stopUnusedServices(), { debug }); + log(`onTaskFinished: checking to stop unused services. hasPending: ${appTaskManager.hasPendingTasks()}`) + if (!appTaskManager.hasPendingTasks()) safe(services.stopUnusedServices(), { debug: log }); } async function getCount() { @@ -1452,7 +1452,7 @@ async function startExec(app, execId, options) { // there is a race where resizing too early results in a 404 "no such exec" // https://git.cloudron.io/cloudron/box/issues/549 setTimeout(async function () { - await safe(docker.resizeExec(execId, { h: options.rows, w: options.columns }, { debug })); + await safe(docker.resizeExec(execId, { h: options.rows, w: options.columns }, { debug: log })); }, 2000); } @@ -1568,7 +1568,7 @@ async function uploadFile(app, sourceFilePath, destFilePath) { // the built-in bash printf understands "%q" but not /usr/bin/printf. // ' gets replaced with '\'' . the first closes the quote and last one starts a new one const escapedDestFilePath = await shell.bash(`printf %q '${destFilePath.replace(/'/g, '\'\\\'\'')}'`, { encoding: 'utf8' }); - debug(`uploadFile: ${sourceFilePath} -> ${escapedDestFilePath}`); + log(`uploadFile: ${sourceFilePath} -> ${escapedDestFilePath}`); const execId = await createExec(app, { cmd: [ 'bash', '-c', `cat - > ${escapedDestFilePath}` ], tty: false }); const destStream = await startExec(app, execId, { tty: false }); @@ -1702,9 +1702,9 @@ async function scheduleTask(appId, installationState, taskId, auditSource) { const options = { timeout: 20 * 60 * 60 * 1000 /* 20 hours */, nice: 15, memoryLimit }; appTaskManager.scheduleTask(appId, taskId, options, async function (error) { - debug(`scheduleTask: task ${taskId} of ${appId} completed. error: %o`, error); + log(`scheduleTask: task ${taskId} of ${appId} completed. error: %o`, error); if (error?.code === tasks.ECRASHED || error?.code === tasks.ESTOPPED) { // if task crashed, update the error - debug(`Apptask crashed/stopped: ${error.message}`); + log(`Apptask crashed/stopped: ${error.message}`); const appError = { message: error.message, reason: BoxError.TASK_ERROR, @@ -1713,12 +1713,12 @@ async function scheduleTask(appId, installationState, taskId, auditSource) { taskId, installationState }; - await safe(update(appId, { installationState: ISTATE_ERROR, error: appError, taskId: null }), { debug }); + await safe(update(appId, { installationState: ISTATE_ERROR, error: appError, taskId: null }), { debug: log }); } else if (!(installationState === ISTATE_PENDING_UNINSTALL && !error)) { // clear out taskId except for successful uninstall - await safe(update(appId, { taskId: null }), { debug }); + await safe(update(appId, { taskId: null }), { debug: log }); } - await safe(onTaskFinished(error, appId, installationState, taskId, auditSource), { debug }); // ignore error + await safe(onTaskFinished(error, appId, installationState, taskId, auditSource), { debug: log }); // ignore error }); } @@ -1740,7 +1740,7 @@ async function addTask(appId, installationState, task, auditSource) { if (updateError && updateError.reason === BoxError.NOT_FOUND) throw new BoxError(BoxError.BAD_STATE, 'Another task is scheduled for this app'); // could be because app went away OR a taskId exists if (updateError) throw updateError; - if (scheduleNow) await safe(scheduleTask(appId, installationState, taskId, auditSource), { debug }); // ignore error + if (scheduleNow) await safe(scheduleTask(appId, installationState, taskId, auditSource), { debug: log }); // ignore error return taskId; } @@ -1882,7 +1882,7 @@ async function install(data, auditSource) { if (constants.DEMO && (await getCount() >= constants.DEMO_APP_LIMIT)) throw new BoxError(BoxError.BAD_STATE, 'Too many installed apps, please uninstall a few and try again'); const appId = crypto.randomUUID(); - debug(`Installing app ${appId}`); + log(`Installing app ${appId}`); const app = { accessRestriction, @@ -2566,7 +2566,7 @@ async function exportApp(app, backupSiteId, auditSource) { if (!canBackupApp(app)) throw new BoxError(BoxError.BAD_STATE, 'App cannot be backed up in this state'); const taskId = await tasks.add(`${tasks.TASK_APP_BACKUP_PREFIX}${app.id}`, [ appId, backupSiteId, { snapshotOnly: true } ]); - safe(tasks.startTask(taskId, {}), { debug }); // background + safe(tasks.startTask(taskId, {}), { debug: log }); // background return { taskId }; } @@ -2887,7 +2887,7 @@ async function getBackupDownloadStream(app, backupId) { const stream = await backupSites.storageApi(backupSite).download(backupSite.config, downloadBackup.remotePath); stream.on('error', function(error) { - debug(`getBackupDownloadStream: read stream error: ${error.message}`); + log(`getBackupDownloadStream: read stream error: ${error.message}`); ps.emit('error', new BoxError(BoxError.EXTERNAL_ERROR, error)); }); stream.pipe(ps); @@ -2926,11 +2926,11 @@ async function restoreApps(apps, options, auditSource) { requireNullTaskId: false // ignore existing stale taskId }; - debug(`restoreApps: marking ${app.fqdn} as ${installationState} using restore config ${JSON.stringify(restoreConfig)}`); + log(`restoreApps: marking ${app.fqdn} as ${installationState} using restore config ${JSON.stringify(restoreConfig)}`); const [addTaskError, taskId] = await safe(addTask(app.id, installationState, task, auditSource)); - if (addTaskError) debug(`restoreApps: error marking ${app.fqdn} for restore: ${JSON.stringify(addTaskError)}`); - else debug(`restoreApps: marked ${app.id} as ${installationState} with taskId ${taskId}`); + if (addTaskError) log(`restoreApps: error marking ${app.fqdn} for restore: ${JSON.stringify(addTaskError)}`); + else log(`restoreApps: marked ${app.id} as ${installationState} with taskId ${taskId}`); } } @@ -2945,7 +2945,7 @@ async function configureApps(apps, options, auditSource) { const scheduleNow = !!options.scheduleNow; for (const app of apps) { - debug(`configureApps: marking ${app.fqdn} for reconfigure (scheduleNow: ${scheduleNow})`); + log(`configureApps: marking ${app.fqdn} for reconfigure (scheduleNow: ${scheduleNow})`); const task = { args: {}, @@ -2955,8 +2955,8 @@ async function configureApps(apps, options, auditSource) { }; const [addTaskError, taskId] = await safe(addTask(app.id, ISTATE_PENDING_CONFIGURE, task, auditSource)); - if (addTaskError) debug(`configureApps: error marking ${app.fqdn} for configure: ${JSON.stringify(addTaskError)}`); - else debug(`configureApps: marked ${app.id} for re-configure with taskId ${taskId}`); + if (addTaskError) log(`configureApps: error marking ${app.fqdn} for configure: ${JSON.stringify(addTaskError)}`); + else log(`configureApps: marked ${app.id} for re-configure with taskId ${taskId}`); } } @@ -2972,7 +2972,7 @@ async function restartAppsUsingAddons(changedAddons, auditSource) { apps = apps.filter(app => app.runState !== RSTATE_STOPPED); // don't start stopped apps for (const app of apps) { - debug(`restartAppsUsingAddons: marking ${app.fqdn} for restart`); + log(`restartAppsUsingAddons: marking ${app.fqdn} for restart`); const task = { args: {}, @@ -2981,27 +2981,27 @@ async function restartAppsUsingAddons(changedAddons, auditSource) { // stop apps before updating the databases because postgres will "lock" them preventing import const [stopError] = await safe(docker.stopContainers(app.id)); - if (stopError) debug(`restartAppsUsingAddons: error stopping ${app.fqdn}`, stopError); + if (stopError) log(`restartAppsUsingAddons: error stopping ${app.fqdn}`, stopError); const [addTaskError, taskId] = await safe(addTask(app.id, ISTATE_PENDING_RESTART, task, auditSource)); - if (addTaskError) debug(`restartAppsUsingAddons: error marking ${app.fqdn} for restart: ${JSON.stringify(addTaskError)}`); - else debug(`restartAppsUsingAddons: marked ${app.id} for restart with taskId ${taskId}`); + if (addTaskError) log(`restartAppsUsingAddons: error marking ${app.fqdn} for restart: ${JSON.stringify(addTaskError)}`); + else log(`restartAppsUsingAddons: marked ${app.id} for restart with taskId ${taskId}`); } } async function schedulePendingTasks(auditSource) { assert.strictEqual(typeof auditSource, 'object'); - debug('schedulePendingTasks: scheduling app tasks'); + log('schedulePendingTasks: scheduling app tasks'); const result = await list(); for (const app of result) { if (!app.taskId) continue; // if not in any pending state, do nothing - debug(`schedulePendingTasks: schedule task for ${app.fqdn} ${app.id}: state=${app.installationState},taskId=${app.taskId}`); + log(`schedulePendingTasks: schedule task for ${app.fqdn} ${app.id}: state=${app.installationState},taskId=${app.taskId}`); - await safe(scheduleTask(app.id, app.installationState, app.taskId, auditSource), { debug }); // ignore error + await safe(scheduleTask(app.id, app.installationState, app.taskId, auditSource), { debug: log }); // ignore error } } diff --git a/src/appstore.js b/src/appstore.js index c391ad331..50a26bc59 100644 --- a/src/appstore.js +++ b/src/appstore.js @@ -4,7 +4,7 @@ import backupSites from './backupsites.js'; import BoxError from './boxerror.js'; import constants from './constants.js'; import dashboard from './dashboard.js'; -import debugModule from 'debug'; +import logger from './logger.js'; import domains from './domains.js'; import dockerRegistries from './dockerregistries.js'; import directoryServer from './directoryserver.js'; @@ -23,7 +23,7 @@ import system from './system.js'; import users from './users.js'; import volumes from './volumes.js'; -const debug = debugModule('box:appstore'); +const { log, trace } = logger('appstore'); // These are the default options and will be adjusted once a subscription state is obtained // Keep in sync with appstore/routes/cloudrons.js @@ -129,7 +129,7 @@ async function getSubscription() { if (!token) throw new BoxError(BoxError.LICENSE_ERROR, 'Missing token'); const [stateError, state] = await safe(getState()); - if (stateError) debug('getSubscription: error getting current state', stateError); + if (stateError) log('getSubscription: error getting current state', stateError); const [error, response] = await safe(superagent.post(`${await getApiServerOrigin()}/api/v1/subscription3`) .query({ accessToken: token }) @@ -157,8 +157,8 @@ async function getSubscription() { // cron hook async function checkSubscription() { const [error, result] = await safe(getSubscription()); - if (error) debug('checkSubscription error:', error); - else debug(`checkSubscription: Cloudron ${result.cloudronId} is on the ${result.plan.name} plan.`); + if (error) log('checkSubscription error:', error); + else log(`checkSubscription: Cloudron ${result.cloudronId} is on the ${result.plan.name} plan.`); } function isFreePlan(subscription) { @@ -236,7 +236,7 @@ async function getAppUpdate(app, options) { // do some sanity checks if (!safe.query(updateInfo, 'manifest.version') || semver.gt(curAppVersion, safe.query(updateInfo, 'manifest.version'))) { - debug('Skipping malformed update of app %s version: %s. got %j', app.id, curAppVersion, updateInfo); + log('Skipping malformed update of app %s version: %s. got %j', app.id, curAppVersion, updateInfo); throw new BoxError(BoxError.EXTERNAL_ERROR, `Malformed update: ${response.status} ${response.text}`); } @@ -268,7 +268,7 @@ async function updateCloudron(data) { if (response.status === 401) throw new BoxError(BoxError.LICENSE_ERROR, 'Invalid appstore token'); if (response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, `Bad response: ${response.status} ${response.text}`); - debug(`updateCloudron: Cloudron updated with data ${JSON.stringify(data)}`); + log(`updateCloudron: Cloudron updated with data ${JSON.stringify(data)}`); } async function registerCloudron3() { @@ -277,7 +277,7 @@ async function registerCloudron3() { const token = await settings.get(settings.APPSTORE_API_TOKEN_KEY); if (token) { // when installed using setupToken, this updates the domain record when called during provisioning - debug('registerCloudron3: already registered. Just updating the record.'); + log('registerCloudron3: already registered. Just updating the record.'); await getSubscription(); return await updateCloudron({ domain, version }); } @@ -296,7 +296,7 @@ async function registerCloudron3() { await settings.set(settings.CLOUDRON_ID_KEY, response.body.cloudronId); await settings.set(settings.APPSTORE_API_TOKEN_KEY, response.body.cloudronToken); - debug(`registerCloudron3: Cloudron registered with id ${response.body.cloudronId}`); + log(`registerCloudron3: Cloudron registered with id ${response.body.cloudronId}`); await getSubscription(); } @@ -307,7 +307,7 @@ async function unregister() { } async function unlinkAccount() { - debug('unlinkAccount: Unlinking existing account.'); + log('unlinkAccount: Unlinking existing account.'); if (constants.DEMO) throw new BoxError(BoxError.BAD_STATE, 'Not allowed in demo mode'); @@ -326,7 +326,7 @@ async function downloadManifest(appStoreId, manifest) { const url = await getApiServerOrigin() + '/api/v1/apps/' + id + (version ? '/versions/' + version : ''); - debug(`downloading manifest from ${url}`); + log(`downloading manifest from ${url}`); const [error, response] = await safe(superagent.get(url).timeout(60 * 1000).ok(() => true)); @@ -388,7 +388,7 @@ async function getApp(appId) { async function downloadIcon(appStoreId, version) { const iconUrl = `${await getApiServerOrigin()}/api/v1/apps/${appStoreId}/versions/${version}/icon`; - return await promiseRetry({ times: 10, interval: 5000, debug }, async function () { + return await promiseRetry({ times: 10, interval: 5000, debug: log }, async function () { const [networkError, response] = await safe(superagent.get(iconUrl) .timeout(60 * 1000) .ok(() => true)); diff --git a/src/apptask.js b/src/apptask.js index a001858ae..0f54718b8 100644 --- a/src/apptask.js +++ b/src/apptask.js @@ -9,7 +9,7 @@ import backuptask from './backuptask.js'; import BoxError from './boxerror.js'; import community from './community.js'; import constants from './constants.js'; -import debugModule from 'debug'; +import logger from './logger.js'; import df from './df.js'; import dns from './dns.js'; import docker from './docker.js'; @@ -28,7 +28,7 @@ import services from './services.js'; import shellModule from './shell.js'; import _ from './underscore.js'; -const debug = debugModule('box:apptask'); +const { log, trace } = logger('apptask'); const shell = shellModule('apptask'); const LOGROTATE_CONFIG_EJS = fs.readFileSync(import.meta.dirname + '/logrotate.ejs', { encoding: 'utf8' }), @@ -63,7 +63,7 @@ async function allocateContainerIp(app) { if (app.manifest.id === constants.PROXY_APP_APPSTORE_ID) return; - await promiseRetry({ times: 10, interval: 0, debug }, async function () { + await promiseRetry({ times: 10, interval: 0, debug: log }, async function () { const iprange = iputils.intFromIp(constants.APPS_IPv4_END) - iputils.intFromIp(constants.APPS_IPv4_START); const rnd = Math.floor(Math.random() * iprange); const containerIp = iputils.ipFromInt(iputils.intFromIp(constants.APPS_IPv4_START) + rnd); @@ -92,7 +92,7 @@ async function deleteAppDir(app, options) { const resolvedAppDataDir = stat.isSymbolicLink() ? safe.fs.readlinkSync(appDataDir) : appDataDir; - debug(`deleteAppDir - removing files in ${resolvedAppDataDir}`); + log(`deleteAppDir - removing files in ${resolvedAppDataDir}`); if (safe.fs.existsSync(resolvedAppDataDir)) { const entries = safe.fs.readdirSync(resolvedAppDataDir); @@ -105,7 +105,7 @@ async function deleteAppDir(app, options) { const entryStat = safe.fs.statSync(fullPath); if (entryStat && !entryStat.isDirectory()) { safe.fs.unlinkSync(fullPath); - debug(`deleteAppDir - ${fullPath} ${safe.error?.message || ''}`); + log(`deleteAppDir - ${fullPath} ${safe.error?.message || ''}`); } } } @@ -135,7 +135,7 @@ async function deleteContainers(app, options) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); - debug('deleteContainer: deleting app containers (app, scheduler)'); + log('deleteContainer: deleting app containers (app, scheduler)'); // remove configs that rely on container id await removeLogrotateConfig(app); @@ -149,7 +149,7 @@ async function cleanupLogs(app) { // note that redis container logs are cleaned up by the addon const [error] = await safe(fs.promises.rm(path.join(paths.LOG_DIR, app.id), { force: true, recursive: true })); - if (error) debug('cleanupLogs: cannot cleanup logs: %o', error); + if (error) log('cleanupLogs: cannot cleanup logs: %o', error); } async function verifyManifest(manifest) { @@ -167,10 +167,10 @@ async function downloadIcon(app) { let packageIcon = null; if (app.versionsUrl && app.manifest.iconUrl) { - debug(`downloadIcon: Downloading community icon ${app.manifest.iconUrl}`); + log(`downloadIcon: Downloading community icon ${app.manifest.iconUrl}`); packageIcon = await community.downloadIcon(app.manifest); } else if (app.appStoreId) { - debug(`downloadIcon: Downloading icon of ${app.appStoreId}@${app.manifest.version}`); + log(`downloadIcon: Downloading icon of ${app.appStoreId}@${app.manifest.version}`); packageIcon = await appstore.downloadIcon(app.appStoreId, app.manifest.version); } @@ -251,7 +251,7 @@ async function updateChecklist(app, newChecks, acknowledged = false) { } async function startApp(app) { - debug('startApp: starting container'); + log('startApp: starting container'); if (app.runState === apps.RSTATE_STOPPED) return; @@ -378,7 +378,7 @@ async function createContainer(app) { if (app.manifest.id === constants.PROXY_APP_APPSTORE_ID) return; - debug('createContainer: creating container'); + log('createContainer: creating container'); const container = await docker.createContainer(app); @@ -808,7 +808,7 @@ async function run(appId, args, progressCallback) { const app = await apps.get(appId); - debug(`run: startTask installationState: ${app.installationState} runState: ${app.runState}`); + log(`run: startTask installationState: ${app.installationState} runState: ${app.runState}`); let cmd; @@ -855,19 +855,19 @@ async function run(appId, args, progressCallback) { cmd = updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null }); break; default: - debug('run: apptask launched with invalid command'); + log('run: apptask launched with invalid command'); throw new BoxError(BoxError.INTERNAL_ERROR, 'Unknown install command in apptask:' + app.installationState); } const [error, result] = await safe(cmd); // only some commands like backup return a result if (error) { - debug(`run: app error for state ${app.installationState}: %o`, error); + log(`run: app error for state ${app.installationState}: %o`, error); if (app.installationState === apps.ISTATE_PENDING_UPDATE && error.backupError) { - debug('run: update aborted because backup failed'); - await safe(updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null }, { debug })); + log('run: update aborted because backup failed'); + await safe(updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null }, { debug: log })); } else { - await safe(updateApp(app, { installationState: apps.ISTATE_ERROR, error: makeTaskError(error, app) }), { debug }); + await safe(updateApp(app, { installationState: apps.ISTATE_ERROR, error: makeTaskError(error, app) }), { debug: log }); } throw error; diff --git a/src/apptaskmanager.js b/src/apptaskmanager.js index b0df2158d..bdeabf534 100644 --- a/src/apptaskmanager.js +++ b/src/apptaskmanager.js @@ -1,6 +1,6 @@ import assert from 'node:assert'; import BoxError from './boxerror.js'; -import debugModule from 'debug'; +import logger from './logger.js'; import fs from 'node:fs'; import locks from './locks.js'; import path from 'node:path'; @@ -9,7 +9,7 @@ import safe from 'safetydance'; import scheduler from './scheduler.js'; import tasks from './tasks.js'; -const debug = debugModule('box:apptaskmanager'); +const { log, trace } = logger('apptaskmanager'); const gActiveTasks = {}; // indexed by app id @@ -22,12 +22,12 @@ const DRAIN_TIMER_SECS = 1000; let gDrainTimerId = null; async function drain() { - debug(`drain: ${gPendingTasks.length} apptasks pending`); + log(`drain: ${gPendingTasks.length} apptasks pending`); for (let i = 0; i < gPendingTasks.length; i++) { const space = Object.keys(gActiveTasks).length - TASK_CONCURRENCY; if (space == 0) { - debug('At concurrency limit, cannot drain anymore'); + log('At concurrency limit, cannot drain anymore'); break; } @@ -53,7 +53,7 @@ async function drain() { .catch((error) => { taskError = error; }) .finally(async () => { delete gActiveTasks[appId]; - await safe(onFinished(taskError, taskResult), { debug }); // hasPendingTasks() can now return false + await safe(onFinished(taskError, taskResult), { debug: log }); // hasPendingTasks() can now return false await locks.release(`${locks.TYPE_APP_TASK_PREFIX}${appId}`); await locks.releaseByTaskId(taskId); scheduler.resumeAppJobs(appId); @@ -68,7 +68,7 @@ async function start() { assert.strictEqual(gDrainTimerId, null); assert.strictEqual(gStarted, false); - debug('started'); + log('started'); gStarted = true; if (gPendingTasks.length) gDrainTimerId = setTimeout(drain, DRAIN_TIMER_SECS); diff --git a/src/asynctask.js b/src/asynctask.js index e69fd9546..b5c731acf 100644 --- a/src/asynctask.js +++ b/src/asynctask.js @@ -1,8 +1,8 @@ -import debugModule from 'debug'; +import logger from './logger.js'; import EventEmitter from 'node:events'; import safe from 'safetydance'; -const debug = debugModule('box:asynctask'); +const { log, trace } = logger('asynctask'); // this runs in-process class AsyncTask extends EventEmitter { @@ -20,16 +20,16 @@ class AsyncTask extends EventEmitter { } async start() { // should not throw! - debug(`start: ${this.name} started`); + log(`start: ${this.name} started`); const [error] = await safe(this._run(this.#abortController.signal)); - debug(`start: ${this.name} finished`); + log(`start: ${this.name} finished`); this.emit('done', { errorMessage: error?.message || '' }); this.#abortController = null; } stop() { if (this.#abortController === null) return; // already finished - debug(`stop: ${this.name} . sending abort signal`); + log(`stop: ${this.name} . sending abort signal`); this.#abortController.abort(); } diff --git a/src/backupcleaner.js b/src/backupcleaner.js index e35191069..e53a96439 100644 --- a/src/backupcleaner.js +++ b/src/backupcleaner.js @@ -6,12 +6,12 @@ import backups from './backups.js'; import backupFormats from './backupformats.js'; import backupSites from './backupsites.js'; import constants from './constants.js'; -import debugModule from 'debug'; +import logger from './logger.js'; import moment from 'moment'; import path from 'node:path'; import safe from 'safetydance'; -const debug = debugModule('box:backupcleaner'); +const { log, trace } = logger('backupcleaner'); function applyBackupRetention(allBackups, retention, referencedBackupIds) { assert(Array.isArray(allBackups)); @@ -67,7 +67,7 @@ function applyBackupRetention(allBackups, retention, referencedBackupIds) { } for (const backup of allBackups) { - debug(`applyBackupRetention: ${backup.remotePath} keep/discard: ${backup.keepReason || backup.discardReason || 'unprocessed'}`); + log(`applyBackupRetention: ${backup.remotePath} keep/discard: ${backup.keepReason || backup.discardReason || 'unprocessed'}`); } } @@ -88,21 +88,21 @@ async function removeBackup(site, backup, progressCallback) { } if (removeError) { - debug(`removeBackup: error removing backup ${removeError.message}`); + log(`removeBackup: error removing backup ${removeError.message}`); return; } // remove integrity info const [removeIntegrityError] = await safe(backupSites.storageApi(site).remove(site.config, `${remotePath}.backupinfo`)); - if (removeIntegrityError) debug(`removeBackup: could not remove integrity info: ${removeIntegrityError.message}`); + if (removeIntegrityError) log(`removeBackup: could not remove integrity info: ${removeIntegrityError.message}`); // prune empty directory if possible const [pruneError] = await safe(backupSites.storageApi(site).remove(site.config, path.dirname(remotePath))); - if (pruneError) debug(`removeBackup: unable to prune backup directory ${path.dirname(remotePath)}: ${pruneError.message}`); + if (pruneError) log(`removeBackup: unable to prune backup directory ${path.dirname(remotePath)}: ${pruneError.message}`); const [delError] = await safe(backups.del(backup.id)); - if (delError) debug(`removeBackup: error removing ${backup.id} from database. %o`, delError); - else debug(`removeBackup: removed ${backup.remotePath}`); + if (delError) log(`removeBackup: error removing ${backup.id} from database. %o`, delError); + else log(`removeBackup: removed ${backup.remotePath}`); } async function cleanupAppBackups(site, referencedBackupIds, progressCallback) { @@ -129,7 +129,7 @@ async function cleanupAppBackups(site, referencedBackupIds, progressCallback) { let appBackupsToRemove = []; for (const appId of Object.keys(appBackupsById)) { const appRetention = Object.assign({ keepLatest: allAppIds.includes(appId) }, site.retention); - debug(`cleanupAppBackups: applying retention for appId ${appId} retention: ${JSON.stringify(appRetention)}`); + log(`cleanupAppBackups: applying retention for appId ${appId} retention: ${JSON.stringify(appRetention)}`); applyBackupRetention(appBackupsById[appId], appRetention, referencedBackupIds); appBackupsToRemove = appBackupsToRemove.concat(appBackupsById[appId].filter(b => !b.keepReason)); } @@ -140,7 +140,7 @@ async function cleanupAppBackups(site, referencedBackupIds, progressCallback) { await removeBackup(site, appBackup, progressCallback); // never errors } - debug('cleanupAppBackups: done'); + log('cleanupAppBackups: done'); return removedAppBackupPaths; } @@ -163,7 +163,7 @@ async function cleanupMailBackups(site, referencedBackupIds, progressCallback) { await removeBackup(site, mailBackup, progressCallback); // never errors } - debug('cleanupMailBackups: done'); + log('cleanupMailBackups: done'); return removedMailBackupPaths; } @@ -193,7 +193,7 @@ async function cleanupBoxBackups(site, progressCallback) { await removeBackup(site, boxBackup, progressCallback); } - debug('cleanupBoxBackups: done'); + log('cleanupBoxBackups: done'); return { removedBoxBackupPaths, referencedBackupIds }; } @@ -223,7 +223,7 @@ async function cleanupMissingBackups(site, progressCallback) { await progressCallback({ message: `Removing missing backup ${backup.remotePath}`}); const [delError] = await safe(backups.del(backup.id)); - if (delError) debug(`cleanupMissingBackups: error removing ${backup.id} from database. %o`, delError); + if (delError) log(`cleanupMissingBackups: error removing ${backup.id} from database. %o`, delError); missingBackupPaths.push(backup.remotePath); } @@ -231,7 +231,7 @@ async function cleanupMissingBackups(site, progressCallback) { ++ page; } while (result.length === perPage); - debug('cleanupMissingBackups: done'); + log('cleanupMissingBackups: done'); return missingBackupPaths; } @@ -242,7 +242,7 @@ async function removeOldAppSnapshots(site) { const snapshotInfo = await backupSites.getSnapshotInfo(site); - const progressCallback = (progress) => { debug(`removeOldAppSnapshots: ${progress.message}`); }; + const progressCallback = (progress) => { log(`removeOldAppSnapshots: ${progress.message}`); }; for (const appId of Object.keys(snapshotInfo)) { if (appId === 'box' || appId === 'mail') continue; @@ -253,16 +253,16 @@ async function removeOldAppSnapshots(site) { const ext = backupFormats.api(site.format).getFileExtension(!!site.encryption); const remotePath = `snapshot/app_${appId}${ext}`; if (ext) { - await safe(backupSites.storageApi(site).remove(site.config, remotePath), { debug }); + await safe(backupSites.storageApi(site).remove(site.config, remotePath), { debug: log }); } else { - await safe(backupSites.storageApi(site).removeDir(site.config, site.limits, remotePath, progressCallback), { debug }); + await safe(backupSites.storageApi(site).removeDir(site.config, site.limits, remotePath, progressCallback), { debug: log }); } await backupSites.setSnapshotInfo(site, appId, null /* info */); - debug(`removeOldAppSnapshots: removed snapshot of app ${appId}`); + log(`removeOldAppSnapshots: removed snapshot of app ${appId}`); } - debug('removeOldAppSnapshots: done'); + log('removeOldAppSnapshots: done'); } async function run(siteId, progressCallback) { @@ -272,14 +272,14 @@ async function run(siteId, progressCallback) { const site = await backupSites.get(siteId); if (!site) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Target not found'); - debug(`run: retention is ${JSON.stringify(site.retention)}`); + log(`run: retention is ${JSON.stringify(site.retention)}`); const status = await backupSites.ensureMounted(site); - debug(`run: mount point status is ${JSON.stringify(status)}`); + log(`run: mount point status is ${JSON.stringify(status)}`); if (status.state !== 'active') throw new BoxError(BoxError.MOUNT_ERROR, `Backup endpoint is not mounted: ${status.message}`); if (site.retention.keepWithinSecs < 0) { - debug('run: keeping all backups'); + log('run: keeping all backups'); return {}; } diff --git a/src/backupformat/rsync.js b/src/backupformat/rsync.js index 443df3bd3..c5680e653 100644 --- a/src/backupformat/rsync.js +++ b/src/backupformat/rsync.js @@ -3,7 +3,7 @@ import async from 'async'; import backupSites from '../backupsites.js'; import BoxError from '../boxerror.js'; import DataLayout from '../datalayout.js'; -import debugModule from 'debug'; +import logger from '../logger.js'; import hush from '../hush.js'; const { DecryptStream, EncryptStream } = hush; import fs from 'node:fs'; @@ -19,7 +19,7 @@ import syncer from '../syncer.js'; import util from 'node:util'; import { Writable } from 'node:stream'; -const debug = debugModule('box:backupformat/rsync'); +const { log, trace } = logger('backupformat/rsync'); const shell = shellModule('backupformat/rsync'); async function addFile(sourceFile, encryption, uploader, progressCallback) { @@ -32,7 +32,7 @@ async function addFile(sourceFile, encryption, uploader, progressCallback) { // destinations dirs/file which are owned by root (this process id) and cannot be copied (run as normal user) const [openError, sourceHandle] = await safe(fs.promises.open(sourceFile, 'r')); if (openError) { - debug(`addFile: ignoring disappeared file: ${sourceFile}`); + log(`addFile: ignoring disappeared file: ${sourceFile}`); return { integrity: null, stats: { transferred: 0 } }; } @@ -61,7 +61,7 @@ async function addFile(sourceFile, encryption, uploader, progressCallback) { const [error] = await safe(pipelinePromise); if (error && !error.message.includes('ENOENT')) throw new BoxError(BoxError.EXTERNAL_ERROR, `tarPack pipeline error for ${sourceFile}: ${error.message}`); // ignore error if file disappears - // debug(`addFile: pipeline finished: ${JSON.stringify(ps.stats())}`); + // log(`addFile: pipeline finished: ${JSON.stringify(ps.stats())}`); await uploader.finish(); @@ -112,7 +112,7 @@ async function restoreFsMetadata(dataLayout, metadataFile) { assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout'); assert.strictEqual(typeof metadataFile, 'string'); - debug(`Recreating empty directories in ${dataLayout.toString()}`); + log(`Recreating empty directories in ${dataLayout.toString()}`); const metadataJson = safe.fs.readFileSync(metadataFile, 'utf8'); if (metadataJson === null) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Error loading fsmetadata.json:' + safe.error.message); @@ -164,7 +164,7 @@ async function sync(backupSite, remotePath, dataLayout, progressCallback) { const concurrency = backupSite.limits?.syncConcurrency || (backupSite.provider === 's3' ? 20 : 10); const cacheFile = path.join(paths.BACKUP_INFO_DIR, backupSite.id, `${dataLayout.getBasename()}.sync.cache`); const { delQueue, addQueue, integrityMap } = await syncer.sync(dataLayout, cacheFile); // integrityMap is unchanged files - debug(`sync: processing ${delQueue.length} deletes, ${addQueue.length} additions and ${integrityMap.size} unchanged`); + log(`sync: processing ${delQueue.length} deletes, ${addQueue.length} additions and ${integrityMap.size} unchanged`); const aggregatedStats = { transferred: 0, size: [...integrityMap.values()].reduce((sum, integrity) => sum + (integrity?.size || 0), 0), // integrity can be null if file had disappeared during upload @@ -193,9 +193,9 @@ async function sync(backupSite, remotePath, dataLayout, progressCallback) { } else if (change.operation === 'remove') { await backupSites.storageApi(backupSite).remove(backupSite.config, fullPath); } else if (change.operation === 'add') { - await promiseRetry({ times: 5, interval: 20000, debug }, async (retryCount) => { + await promiseRetry({ times: 5, interval: 20000, debug: log }, async (retryCount) => { reportUploadProgress(`Current: ${change.path}` + (retryCount > 1 ? ` (Try ${retryCount})` : '')); - if (retryCount > 1) debug(`sync: retrying ${change.path} position ${change.position} try ${retryCount}`); + if (retryCount > 1) log(`sync: retrying ${change.path} position ${change.position} try ${retryCount}`); const uploader = await backupSites.storageApi(backupSite).upload(backupSite.config, backupSite.limits, fullPath); const result = await addFile(dataLayout.toLocalPath('./' + change.path), backupSite.encryption, uploader, (progress) => { reportUploadProgress(progress.message); @@ -210,11 +210,11 @@ async function sync(backupSite, remotePath, dataLayout, progressCallback) { } const [delError] = await safe(async.eachLimit(delQueue, concurrency, async (change) => await processSyncerChange(change, backupSite, remotePath, dataLayout, progressCallback))); - debug('sync: done processing deletes. error: %o', delError); + log('sync: done processing deletes. error: %o', delError); if (delError) throw delError; const [addError] = await safe(async.eachLimit(addQueue, concurrency, async (change) => await processSyncerChange(change, backupSite, remotePath, dataLayout, progressCallback))); - debug('sync: done processing adds. error: %o', addError); + log('sync: done processing adds. error: %o', addError); if (addError) throw addError; progressCallback({ message: `Uploaded ${completedAdds} files` }); @@ -235,7 +235,7 @@ async function downloadDir(backupSite, remotePath, dataLayout, progressCallback) const encryptedFilenames = backupSite.encryption?.encryptedFilenames || false; - debug(`downloadDir: ${remotePath} to ${dataLayout.toString()}. encryption filenames: ${encryptedFilenames}. encrypted files: ${!!backupSite.encryption}`); + log(`downloadDir: ${remotePath} to ${dataLayout.toString()}. encryption filenames: ${encryptedFilenames}. encrypted files: ${!!backupSite.encryption}`); let completedFiles = 0, totalFiles = 0; let lastProgressTime = 0; @@ -264,7 +264,7 @@ async function downloadDir(backupSite, remotePath, dataLayout, progressCallback) const [downloadError, sourceStream] = await safe(backupSites.storageApi(backupSite).download(backupSite.config, entry.path)); if (downloadError) { - debug(`downloadDir: download ${entry.path} to ${destFilePath} errored: ${downloadError.message}`); + log(`downloadDir: download ${entry.path} to ${destFilePath} errored: ${downloadError.message}`); throw downloadError; } @@ -290,7 +290,7 @@ async function downloadDir(backupSite, remotePath, dataLayout, progressCallback) const [pipelineError] = await safe(pipeline(streams)); if (pipelineError) { - debug(`downloadDir: download error ${entry.path} to ${destFilePath}: ${pipelineError.message}`); + log(`downloadDir: download error ${entry.path} to ${destFilePath}: ${pipelineError.message}`); throw pipelineError; } }); @@ -318,7 +318,7 @@ async function download(backupSite, remotePath, dataLayout, progressCallback) { assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout'); assert.strictEqual(typeof progressCallback, 'function'); - debug(`download: Downloading ${remotePath} to ${dataLayout.toString()}`); + log(`download: Downloading ${remotePath} to ${dataLayout.toString()}`); await downloadDir(backupSite, remotePath, dataLayout, progressCallback); await restoreFsMetadata(dataLayout, `${dataLayout.localRoot()}/fsmetadata.json`); @@ -330,7 +330,7 @@ async function upload(backupSite, remotePath, dataLayout, progressCallback) { assert.strictEqual(typeof dataLayout, 'object'); assert.strictEqual(typeof progressCallback, 'function'); - debug(`upload: uploading to site ${backupSite.id} path ${remotePath} (encrypted: ${!!backupSite.encryption}) dataLayout ${dataLayout.toString()}`); + log(`upload: uploading to site ${backupSite.id} path ${remotePath} (encrypted: ${!!backupSite.encryption}) dataLayout ${dataLayout.toString()}`); await saveFsMetadata(dataLayout, `${dataLayout.localRoot()}/fsmetadata.json`); return await sync(backupSite, remotePath, dataLayout, progressCallback); // { stats, integrityMap } @@ -370,7 +370,7 @@ async function verify(backupSite, remotePath, integrityMap, progressCallback) { assert(util.types.isMap(integrityMap), 'integrityMap should be a Map'); assert.strictEqual(typeof progressCallback, 'function'); - debug(`verify: Verifying ${remotePath}`); + log(`verify: Verifying ${remotePath}`); // https://www.digitalocean.com/community/questions/rate-limiting-on-spaces?answer=40441 const concurrency = backupSite.limits?.downloadConcurrency || (backupSite.provider === 's3' ? 30 : 10); @@ -386,12 +386,12 @@ async function verify(backupSite, remotePath, integrityMap, progressCallback) { const integrity = integrityMap.get(relativePath); if (result.transferred !== integrity.size) { messages.push(`${entry.path} has size ${result.transferred}. Expecting ${integrity.size}`); - debug(`verify: size check of ${entry.path} failed: ${messages.at(-1)}`); + log(`verify: size check of ${entry.path} failed: ${messages.at(-1)}`); } else if (result.digest !== integrity.sha256) { messages.push(`${entry.path} has digest ${result.digest}. Expecting ${integrity.sha256}`); - debug(`verify: digest check of ${entry.path} failed: ${messages.at(-1)}`); + log(`verify: digest check of ${entry.path} failed: ${messages.at(-1)}`); } else { - debug(`verify: ${entry.path} passed`); + log(`verify: ${entry.path} passed`); } }); fileCount += batch.entries.length; @@ -401,7 +401,7 @@ async function verify(backupSite, remotePath, integrityMap, progressCallback) { if (integrityMap.size !== fileCount) { messages.push(`Got ${fileCount} files. Expecting ${integrityMap.size} files`); - debug(`verify: file count mismatch: ${messages.at(-1)}`); + log(`verify: file count mismatch: ${messages.at(-1)}`); } return messages; diff --git a/src/backupformat/tgz.js b/src/backupformat/tgz.js index 3b5fcaa3e..6365a59e3 100644 --- a/src/backupformat/tgz.js +++ b/src/backupformat/tgz.js @@ -2,7 +2,7 @@ import assert from 'node:assert'; import backupSites from '../backupsites.js'; import BoxError from '../boxerror.js'; import DataLayout from '../datalayout.js'; -import debugModule from 'debug'; +import logger from '../logger.js'; import hush from '../hush.js'; const { DecryptStream, EncryptStream } = hush; import fs from 'node:fs'; @@ -17,7 +17,7 @@ import tar from 'tar-stream'; import util from 'node:util'; import zlib from 'node:zlib'; -const debug = debugModule('box:backupformat/tgz'); +const { log, trace } = logger('backupformat/tgz'); // In tar, the entry header contains the file size. If we don't provide it those many bytes, the tar will become corrupt // Linux provides no guarantee of how many bytes can be read from a file. This is the case with sqlite and log files @@ -31,12 +31,12 @@ class EnsureFileSizeStream extends Transform { _transform(chunk, encoding, callback) { if (this._remaining <= 0) { - debug(`EnsureFileSizeStream: ${this._name} dropping ${chunk.length} bytes`); + log(`EnsureFileSizeStream: ${this._name} dropping ${chunk.length} bytes`); return callback(null); } if (this._remaining - chunk.length < 0) { - debug(`EnsureFileSizeStream: ${this._name} dropping extra ${chunk.length - this._remaining} bytes`); + log(`EnsureFileSizeStream: ${this._name} dropping extra ${chunk.length - this._remaining} bytes`); chunk = chunk.subarray(0, this._remaining); this._remaining = 0; } else { @@ -48,7 +48,7 @@ class EnsureFileSizeStream extends Transform { _flush(callback) { if (this._remaining > 0) { - debug(`EnsureFileSizeStream: ${this._name} injecting ${this._remaining} bytes`); + log(`EnsureFileSizeStream: ${this._name} injecting ${this._remaining} bytes`); this.push(Buffer.alloc(this._remaining, 0)); } callback(); @@ -63,7 +63,7 @@ function addEntryToPack(pack, header, options) { return new Promise((resolve, reject) => { const packEntry = safe(() => pack.entry(header, function (error) { if (error) { - debug(`addToPack: error adding ${header.name} ${header.type} ${error.message}`); + log(`addToPack: error adding ${header.name} ${header.type} ${error.message}`); reject(new BoxError(BoxError.FS_ERROR, error.message)); } else { resolve(); @@ -74,7 +74,7 @@ function addEntryToPack(pack, header, options) { if (options?.input) { const ensureFileSizeStream = new EnsureFileSizeStream({ name: header.name, size: header.size }); - safe(stream.pipeline(options.input, ensureFileSizeStream, packEntry), { debug }); // background. rely on pack.entry callback for promise completion + safe(stream.pipeline(options.input, ensureFileSizeStream, packEntry), { debug: log }); // background. rely on pack.entry callback for promise completion } }); } @@ -92,7 +92,7 @@ async function addPathToPack(pack, localPath, dataLayout) { const dir = queue.shift(); const [readdirError, entries] = await safe(fs.promises.readdir(dir, { withFileTypes: true })); if (!entries) { - debug(`tarPack: skipping directory ${dir}: ${readdirError.message}`); + log(`tarPack: skipping directory ${dir}: ${readdirError.message}`); continue; } const subdirs = []; @@ -101,9 +101,9 @@ async function addPathToPack(pack, localPath, dataLayout) { const headerName = dataLayout.toRemotePath(abspath); if (entry.isFile()) { const [openError, handle] = await safe(fs.promises.open(abspath, 'r')); - if (!handle) { debug(`tarPack: skipping file, could not open ${abspath}: ${openError.message}`); continue; } + if (!handle) { log(`tarPack: skipping file, could not open ${abspath}: ${openError.message}`); continue; } const [statError, stat] = await safe(handle.stat()); - if (!stat) { debug(`tarPack: skipping file, could not stat ${abspath}: ${statError.message}`); continue; } + if (!stat) { log(`tarPack: skipping file, could not stat ${abspath}: ${statError.message}`); continue; } const header = { name: headerName, type: 'file', mode: stat.mode, size: stat.size, uid: process.getuid(), gid: process.getgid() }; if (stat.size > 8589934590 || entry.name.length > 99) header.pax = { size: stat.size }; const input = handle.createReadStream({ autoClose: true }); @@ -116,12 +116,12 @@ async function addPathToPack(pack, localPath, dataLayout) { ++stats.dirCount; } else if (entry.isSymbolicLink()) { const [readlinkError, site] = await safe(fs.promises.readlink(abspath)); - if (!site) { debug(`tarPack: skipping link, could not readlink ${abspath}: ${readlinkError.message}`); continue; } + if (!site) { log(`tarPack: skipping link, could not readlink ${abspath}: ${readlinkError.message}`); continue; } const header = { name: headerName, type: 'symlink', linkname: site, uid: process.getuid(), gid: process.getgid() }; await addEntryToPack(pack, header, { /* options */ }); ++stats.linkCount; } else { - debug(`tarPack: ignoring unknown type ${entry.name} ${entry.type}`); + log(`tarPack: ignoring unknown type ${entry.name} ${entry.type}`); } } @@ -163,11 +163,11 @@ async function tarPack(dataLayout, encryption, uploader, progressCallback) { let fileCount = 0; for (const localPath of dataLayout.localPaths()) { - const [error, stats] = await safe(addPathToPack(pack, localPath, dataLayout), { debug }); + const [error, stats] = await safe(addPathToPack(pack, localPath, dataLayout), { debug: log }); if (error) break; // the pipeline will error and we will retry the whole packing all over fileCount += stats.fileCount; } - debug(`tarPack: packed ${fileCount} files`); + log(`tarPack: packed ${fileCount} files`); pack.finalize(); // harmless to call if already in error state @@ -175,7 +175,7 @@ async function tarPack(dataLayout, encryption, uploader, progressCallback) { if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, `tarPack pipeline error: ${error.message}`); const stats = ps.stats(); // { startTime, totalMsecs, transferred } - debug(`tarPack: pipeline finished: ${JSON.stringify(stats)}`); + log(`tarPack: pipeline finished: ${JSON.stringify(stats)}`); await uploader.finish(); return { @@ -195,7 +195,7 @@ async function tarExtract(inStream, dataLayout, encryption, progressCallback) { let entryCount = 0; extract.on('entry', async function (header, entryStream, next) { if (path.isAbsolute(header.name)) { - debug(`tarExtract: ignoring absolute path ${header.name}`); + log(`tarExtract: ignoring absolute path ${header.name}`); return next(); } ++entryCount; @@ -211,7 +211,7 @@ async function tarExtract(inStream, dataLayout, encryption, progressCallback) { await safe(fs.promises.unlink(abspath)); // remove any link created from previous failed extract [error] = await safe(fs.promises.symlink(header.linkname, abspath)); } else { - debug(`tarExtract: ignoring unknown entry: ${header.name} ${header.type}`); + log(`tarExtract: ignoring unknown entry: ${header.name} ${header.type}`); entryStream.resume(); // drain } @@ -220,7 +220,7 @@ async function tarExtract(inStream, dataLayout, encryption, progressCallback) { [error] = await safe(fs.promises.lutimes(abspath, now /* atime */, header.mtime)); // for dirs, mtime will get overwritten next(error); }); - extract.on('finish', () => debug(`tarExtract: extracted ${entryCount} entries`)); + extract.on('finish', () => log(`tarExtract: extracted ${entryCount} entries`)); const gunzip = zlib.createGunzip({}); const ps = new ProgressStream({ interval: 10000 }); @@ -242,7 +242,7 @@ async function tarExtract(inStream, dataLayout, encryption, progressCallback) { if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, `tarExtract pipeline error: ${error.message}`); } - debug(`tarExtract: pipeline finished: ${JSON.stringify(ps.stats())}`); + log(`tarExtract: pipeline finished: ${JSON.stringify(ps.stats())}`); } async function download(backupSite, remotePath, dataLayout, progressCallback) { @@ -251,9 +251,9 @@ async function download(backupSite, remotePath, dataLayout, progressCallback) { assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout'); assert.strictEqual(typeof progressCallback, 'function'); - debug(`download: Downloading ${remotePath} to ${dataLayout.toString()}`); + log(`download: Downloading ${remotePath} to ${dataLayout.toString()}`); - await promiseRetry({ times: 3, interval: 20000, debug }, async () => { + await promiseRetry({ times: 3, interval: 20000, debug: log }, async () => { progressCallback({ message: `Downloading backup ${remotePath}` }); const sourceStream = await backupSites.storageApi(backupSite).download(backupSite.config, remotePath); @@ -267,9 +267,9 @@ async function upload(backupSite, remotePath, dataLayout, progressCallback) { assert.strictEqual(typeof dataLayout, 'object'); assert.strictEqual(typeof progressCallback, 'function'); - debug(`upload: uploading to site ${backupSite.id} path ${remotePath} (encrypted: ${!!backupSite.encryption}) dataLayout ${dataLayout.toString()}`); + log(`upload: uploading to site ${backupSite.id} path ${remotePath} (encrypted: ${!!backupSite.encryption}) dataLayout ${dataLayout.toString()}`); - return await promiseRetry({ times: 5, interval: 20000, debug }, async () => { + return await promiseRetry({ times: 5, interval: 20000, debug: log }, async () => { progressCallback({ message: `Uploading backup ${remotePath}` }); const uploader = await backupSites.storageApi(backupSite).upload(backupSite.config, backupSite.limits, remotePath); @@ -296,7 +296,7 @@ async function verify(backupSite, remotePath, integrityMap, progressCallback) { assert(util.types.isMap(integrityMap), 'integrityMap should be a Map'); assert.strictEqual(typeof progressCallback, 'function'); - debug(`verify: Verifying ${remotePath}`); + log(`verify: Verifying ${remotePath}`); const inStream = await backupSites.storageApi(backupSite).download(backupSite.config, remotePath); @@ -305,17 +305,17 @@ async function verify(backupSite, remotePath, integrityMap, progressCallback) { const extract = tar.extract(); extract.on('entry', async function (header, entryStream, next) { if (path.isAbsolute(header.name)) { - debug(`verify: ignoring absolute path ${header.name}`); + log(`verify: ignoring absolute path ${header.name}`); return next(); } - debug(`verify: ${header.name} ${header.size} ${header.type}`); + log(`verify: ${header.name} ${header.size} ${header.type}`); if (header.type === 'file') { ++fileCount; } entryStream.resume(); // drain next(); }); - extract.on('finish', () => debug('verify: extract finished')); + extract.on('finish', () => log('verify: extract finished')); const hash = new HashStream(); const gunzip = zlib.createGunzip({}); @@ -336,7 +336,7 @@ async function verify(backupSite, remotePath, integrityMap, progressCallback) { } const integrity = integrityMap.get('.'); - debug(`verify: Expecting: ${JSON.stringify(integrity)} Actual: size:${ps.stats().transferred} filecount:${fileCount} digest:${hash.digest()}`); + log(`verify: Expecting: ${JSON.stringify(integrity)} Actual: size:${ps.stats().transferred} filecount:${fileCount} digest:${hash.digest()}`); const messages = []; if (integrity.size !== ps.stats().transferred) messages.push(`Size mismatch. Expected: ${integrity.size} Actual: ${ps.stats().transferred}`); diff --git a/src/backupintegrity.js b/src/backupintegrity.js index 8162b29ea..a0a36247c 100644 --- a/src/backupintegrity.js +++ b/src/backupintegrity.js @@ -5,10 +5,10 @@ import backupSites from './backupsites.js'; import BoxError from './boxerror.js'; import consumers from 'node:stream/consumers'; import crypto from 'node:crypto'; -import debugModule from 'debug'; +import logger from './logger.js'; import safe from 'safetydance'; -const debug = debugModule('box:backupintegrity'); +const { log, trace } = logger('backupintegrity'); async function downloadBackupInfo(backupSite, backup) { @@ -47,7 +47,7 @@ async function verify(backup, backupSite, progressCallback) { if (verifyError) messages.push(`Failed to verify ${backup.remotePath}: ${verifyError.message}`); if (verifyMessages) messages.push(...verifyMessages); - debug(`verified: ${backup.remotePath} ${JSON.stringify(messages, null, 4)}`); + log(`verified: ${backup.remotePath} ${JSON.stringify(messages, null, 4)}`); stats.duration = Date.now() - stats.startTime; return { stats, messages: messages.slice(0, 50) }; // keep rsync fails to 50 to not overflow db diff --git a/src/backups.js b/src/backups.js index 7b1681a7b..5bbcfc0d4 100644 --- a/src/backups.js +++ b/src/backups.js @@ -1,13 +1,13 @@ import assert from 'node:assert'; import BoxError from './boxerror.js'; import database from './database.js'; -import debugModule from 'debug'; +import logger from './logger.js'; import eventlog from './eventlog.js'; import hat from './hat.js'; import safe from 'safetydance'; import tasks from './tasks.js'; -const debug = debugModule('box:backups'); +const { log, trace } = logger('backups'); const BACKUP_TYPE_APP = 'app'; const BACKUP_STATE_NORMAL = 'normal'; @@ -222,11 +222,11 @@ async function startIntegrityCheck(backup, auditSource) { // background tasks.startTask(taskId, {}) .then(async (status) => { - debug(`startIntegrityCheck: task completed`); + log(`startIntegrityCheck: task completed`); await eventlog.add(eventlog.ACTION_BACKUP_INTEGRITY_FINISH, auditSource, { status, taskId, backupId: backup.id }); }) .catch(async (error) => { - debug(`startIntegrityCheck: task error. ${error.message}`); + log(`startIntegrityCheck: task error. ${error.message}`); await eventlog.add(eventlog.ACTION_BACKUP_INTEGRITY_FINISH, auditSource, { errorMessage: error.message, taskId, backupId: backup.id }); }); diff --git a/src/backupsites.js b/src/backupsites.js index 2644e83a3..60e56b651 100644 --- a/src/backupsites.js +++ b/src/backupsites.js @@ -6,7 +6,7 @@ import cron from './cron.js'; import { CronTime } from 'cron'; import crypto from 'node:crypto'; import database from './database.js'; -import debugModule from 'debug'; +import logger from './logger.js'; import eventlog from './eventlog.js'; import hush from './hush.js'; import locks from './locks.js'; @@ -18,7 +18,7 @@ import storageFilesystem from './storage/filesystem.js'; import storageS3 from './storage/s3.js'; import storageGcs from './storage/gcs.js'; -const debug = debugModule('box:backups'); +const { log, trace } = logger('backups'); // format: rsync or tgz @@ -304,7 +304,7 @@ async function del(backupSite, auditSource) { assert.strictEqual(typeof backupSite, 'object'); assert.strictEqual(typeof auditSource, 'object'); - await safe(storageApi(backupSite).teardown(backupSite.config), { debug }); // ignore error + await safe(storageApi(backupSite).teardown(backupSite.config), { debug: log }); // ignore error const queries = [ { query: 'DELETE FROM archives WHERE backupId IN (SELECT id FROM backups WHERE siteId=?)', args: [ backupSite.id ] }, @@ -437,12 +437,12 @@ async function setConfig(backupSite, newConfig, auditSource) { newConfig = structuredClone(newConfig); // make a copy storageApi(backupSite).injectPrivateFields(newConfig, oldConfig); - debug('setConfig: validating new storage configuration'); + log('setConfig: validating new storage configuration'); const sanitizedConfig = await storageApi(backupSite).verifyConfig({ id: backupSite.id, provider: backupSite.provider, config: newConfig }); await update(backupSite, { config: sanitizedConfig }); - debug('setConfig: setting up new storage configuration'); + log('setConfig: setting up new storage configuration'); await storageApi(backupSite).setup(sanitizedConfig); await eventlog.add(eventlog.ACTION_BACKUP_SITE_UPDATE, auditSource, { name: backupSite.name, config: storageApi(backupSite).removePrivateFields(newConfig) }); @@ -487,13 +487,13 @@ async function add(data, auditSource) { const id = crypto.randomUUID(); if (!safe.fs.mkdirSync(`${paths.BACKUP_INFO_DIR}/${id}`)) throw new BoxError(BoxError.FS_ERROR, `Failed to create info dir: ${safe.error.message}`); - debug('add: validating new storage configuration'); + log('add: validating new storage configuration'); const sanitizedConfig = await storageApi({ provider }).verifyConfig({id, provider, config }); await database.query('INSERT INTO backupSites (id, name, provider, configJson, contentsJson, limitsJson, integrityKeyPairJson, retentionJson, schedule, encryptionJson, format, enableForUpdates) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [ id, name, provider, JSON.stringify(sanitizedConfig), JSON.stringify(contents), JSON.stringify(limits), JSON.stringify(integrityKeyPair), JSON.stringify(retention), schedule, JSON.stringify(encryption), format, enableForUpdates ]); - debug('add: setting up new storage configuration'); + log('add: setting up new storage configuration'); await storageApi({ provider }).setup(sanitizedConfig); await eventlog.add(eventlog.ACTION_BACKUP_SITE_ADD, auditSource, { id, name, provider, contents, schedule, format }); @@ -504,7 +504,7 @@ async function add(data, auditSource) { async function addDefault(auditSource) { assert.strictEqual(typeof auditSource, 'object'); - debug('addDefault: adding default backup site'); + log('addDefault: adding default backup site'); const defaultBackupSite = { name: 'Default', provider: 'filesystem', @@ -536,7 +536,7 @@ async function createPseudo(data) { encryption.encryptionPasswordHint = ''; } - debug('add: validating new storage configuration'); + log('add: validating new storage configuration'); const sanitizedConfig = await storageApi({ provider }).verifyConfig({id, provider, config }); return { id, format, provider, config: sanitizedConfig, encryption }; } @@ -547,7 +547,7 @@ async function reinitAll() { if (!safe.fs.mkdirSync(`${paths.BACKUP_INFO_DIR}/${site.id}`, { recursive: true })) throw new BoxError(BoxError.FS_ERROR, `Failed to create info dir: ${safe.error.message}`); const status = await getStatus(site); if (status.state === 'active') continue; - safe(remount(site), { debug }); // background + safe(remount(site), { debug: log }); // background } } diff --git a/src/backuptask.js b/src/backuptask.js index b8a3fa6e1..e329a1167 100644 --- a/src/backuptask.js +++ b/src/backuptask.js @@ -8,7 +8,7 @@ import constants from './constants.js'; import crypto from 'node:crypto'; import DataLayout from './datalayout.js'; import database from './database.js'; -import debugModule from 'debug'; +import logger from './logger.js'; import df from './df.js'; import locks from './locks.js'; import path from 'node:path'; @@ -20,7 +20,7 @@ import shellModule from './shell.js'; import stream from 'stream/promises'; import util from 'util'; -const debug = debugModule('box:backuptask'); +const { log, trace } = logger('backuptask'); const shell = shellModule('backuptask'); @@ -40,22 +40,22 @@ async function checkPreconditions(backupSite, dataLayout) { // check mount status before uploading const status = await backupSites.ensureMounted(backupSite); - debug(`checkPreconditions: mount point status is ${JSON.stringify(status)}`); + log(`checkPreconditions: mount point status is ${JSON.stringify(status)}`); if (status.state !== 'active') throw new BoxError(BoxError.MOUNT_ERROR, `Backup endpoint is not active: ${status.message}`); // check availabe size. this requires root for df to work const available = await backupSites.storageApi(backupSite).getAvailableSize(backupSite.config); let used = 0; for (const localPath of dataLayout.localPaths()) { - debug(`checkPreconditions: getting disk usage of ${localPath}`); + log(`checkPreconditions: getting disk usage of ${localPath}`); // du can error when files go missing as it is computing the size. it still prints some size anyway // to match df output in getAvailableSize() we must use disk usage size here and not apparent size const [duError, result] = await safe(shell.spawn('du', [ '--dereference-args', '--summarize', '--block-size=1', '--exclude=*.lock', '--exclude=dovecot.list.index.log.*', localPath], { encoding: 'utf8' })); - if (duError) debug(`checkPreconditions: du error for ${localPath}. code: ${duError.code} stderror: ${duError.stderr}`); + if (duError) log(`checkPreconditions: du error for ${localPath}. code: ${duError.code} stderror: ${duError.stderr}`); used += parseInt(duError ? duError.stdout : result, 10); } - debug(`checkPreconditions: total required=${used} available=${available}`); + log(`checkPreconditions: total required=${used} available=${available}`); const needed = 0.6 * used + (1024 * 1024 * 1024); // check if there is atleast 1GB left afterwards. aim for 60% because rsync/tgz won't need full 100% if (available <= needed) throw new BoxError(BoxError.FS_ERROR, `Not enough disk space for backup. Needed: ${df.prettyBytes(needed)} Available: ${df.prettyBytes(available)}`); @@ -81,7 +81,7 @@ async function upload(remotePath, siteId, dataLayoutString, progressCallback) { assert.strictEqual(typeof dataLayoutString, 'string'); assert.strictEqual(typeof progressCallback, 'function'); - debug(`upload: path ${remotePath} site ${siteId} dataLayout ${dataLayoutString}`); + log(`upload: path ${remotePath} site ${siteId} dataLayout ${dataLayoutString}`); const backupSite = await backupSites.get(siteId); if (!backupSite) throw new BoxError(BoxError.NOT_FOUND, 'Backup site not found'); @@ -99,7 +99,7 @@ async function upload(remotePath, siteId, dataLayoutString, progressCallback) { // - rsync: size (final backup size) will be different from what was transferred (only changed files) // stats.fileCount and stats.size are stored in db and should match up what is written into .backupinfo const { stats, integrityMap } = await backupFormats.api(backupSite.format).upload(backupSite, remotePath, dataLayout, progressCallback); - debug(`upload: path ${remotePath} site ${siteId} uploaded: ${JSON.stringify(stats)}`); + log(`upload: path ${remotePath} site ${siteId} uploaded: ${JSON.stringify(stats)}`); progressCallback({ message: `Uploading integrity information to ${remotePath}.backupinfo` }); const signature = await uploadBackupInfo(backupSite, remotePath, integrityMap); @@ -112,7 +112,7 @@ async function download(backupSite, remotePath, dataLayout, progressCallback) { assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout'); assert.strictEqual(typeof progressCallback, 'function'); - debug(`download: Downloading ${remotePath} of format ${backupSite.format} (encrypted: ${!!backupSite.encryption}) to ${dataLayout.toString()}`); + log(`download: Downloading ${remotePath} of format ${backupSite.format} (encrypted: ${!!backupSite.encryption}) to ${dataLayout.toString()}`); await backupFormats.api(backupSite.format).download(backupSite, remotePath, dataLayout, progressCallback); } @@ -128,10 +128,10 @@ async function restore(backupSite, remotePath, progressCallback) { await download(backupSite, remotePath, dataLayout, progressCallback); - debug('restore: download completed, importing database'); + log('restore: download completed, importing database'); await database.importFromFile(`${dataLayout.localRoot()}/box.mysqldump`); - debug('restore: database imported'); + log('restore: database imported'); await locks.releaseAll(); // clear the locks table in database } @@ -155,7 +155,7 @@ async function downloadApp(app, restoreConfig, progressCallback) { } await download(backupSite, remotePath, dataLayout, progressCallback); - debug('downloadApp: time: %s', (new Date() - startTime)/1000); + log('downloadApp: time: %s', (new Date() - startTime)/1000); } async function runBackupUpload(uploadConfig, progressCallback) { @@ -172,21 +172,21 @@ async function runBackupUpload(uploadConfig, progressCallback) { const envCopy = Object.assign({}, process.env); if (backupSite.limits?.memoryLimit >= 2*1024*1024*1024) { const heapSize = Math.min((backupSite.limits.memoryLimit/1024/1024) - 256, 8192); - debug(`runBackupUpload: adjusting heap size to ${heapSize}M`); + log(`runBackupUpload: adjusting heap size to ${heapSize}M`); envCopy.NODE_OPTIONS = `--max-old-space-size=${heapSize}`; } let lastMessage = null; // the script communicates error result as a string function onMessage(progress) { // this is { message } or { result } if ('message' in progress) return progressCallback({ message: `${progress.message} (${progressTag})` }); - debug(`runBackupUpload: result - ${JSON.stringify(progress)}`); + log(`runBackupUpload: result - ${JSON.stringify(progress)}`); lastMessage = progress; } // do not use debug for logging child output because it already has timestamps via it's own debug const [error] = await safe(shell.sudo([ BACKUP_UPLOAD_CMD, remotePath, backupSite.id, dataLayout.toString() ], { env: envCopy, preserveEnv: true, onMessage, logger: process.stdout.write })); if (error && (error.code === null /* signal */ || (error.code !== 0 && error.code !== 50))) { // backuptask crashed - debug(`runBackupUpload: backuptask crashed`, error); + log(`runBackupUpload: backuptask crashed`, error); throw new BoxError(BoxError.INTERNAL_ERROR, 'Backuptask crashed'); } else if (error && error.code === 50) { // exited with error throw new BoxError(BoxError.EXTERNAL_ERROR, lastMessage.errorMessage); @@ -203,7 +203,7 @@ async function snapshotBox(progressCallback) { const startTime = new Date(); await database.exportToFile(`${paths.BOX_DATA_DIR}/box.mysqldump`); - debug(`snapshotBox: took ${(new Date() - startTime)/1000} seconds`); + log(`snapshotBox: took ${(new Date() - startTime)/1000} seconds`); } async function uploadBoxSnapshot(backupSite, progressCallback) { @@ -230,7 +230,7 @@ async function uploadBoxSnapshot(backupSite, progressCallback) { const { stats, integrity } = await runBackupUpload(uploadConfig, progressCallback); - debug(`uploadBoxSnapshot: took ${(new Date() - startTime)/1000} seconds`); + log(`uploadBoxSnapshot: took ${(new Date() - startTime)/1000} seconds`); await backupSites.setSnapshotInfo(backupSite, 'box', { timestamp: new Date().toISOString() }); @@ -246,17 +246,17 @@ async function copy(backupSite, srcRemotePath, destRemotePath, progressCallback) const startTime = new Date(); const [copyError] = await safe(backupFormats.api(backupSite.format).copy(backupSite, srcRemotePath, destRemotePath, progressCallback)); if (copyError) { - debug(`copy: copy to ${destRemotePath} errored. error: ${copyError.message}`); + log(`copy: copy to ${destRemotePath} errored. error: ${copyError.message}`); throw copyError; } - debug(`copy: copied successfully to ${destRemotePath}. Took ${(new Date() - startTime)/1000} seconds`); + log(`copy: copied successfully to ${destRemotePath}. Took ${(new Date() - startTime)/1000} seconds`); const [copyChecksumError] = await safe(backupSites.storageApi(backupSite).copy(backupSite.config, `${srcRemotePath}.backupinfo`, `${destRemotePath}.backupinfo`, progressCallback)); if (copyChecksumError) { - debug(`copy: copy to ${destRemotePath} errored. error: ${copyChecksumError.message}`); + log(`copy: copy to ${destRemotePath} errored. error: ${copyChecksumError.message}`); throw copyChecksumError; } - debug(`copy: copied backupinfo successfully to ${destRemotePath}.backupinfo`); + log(`copy: copied backupinfo successfully to ${destRemotePath}.backupinfo`); } async function backupBox(backupSite, appBackupsMap, tag, options, progressCallback) { @@ -281,7 +281,7 @@ async function backupBox(backupSite, appBackupsMap, tag, options, progressCallba duration: acc.duration + cur.upload.duration, }), stats.upload); - debug(`backupBox: rotating box snapshot of ${backupSite.id} to id ${remotePath}. ${JSON.stringify(stats)}`); + log(`backupBox: rotating box snapshot of ${backupSite.id} to id ${remotePath}. ${JSON.stringify(stats)}`); const data = { remotePath, @@ -329,7 +329,7 @@ async function snapshotApp(app, progressCallback) { await services.runBackupCommand(app); await services.backupAddons(app, app.manifest.addons); - debug(`snapshotApp: ${app.fqdn} took ${(new Date() - startTime)/1000} seconds`); + log(`snapshotApp: ${app.fqdn} took ${(new Date() - startTime)/1000} seconds`); } async function uploadAppSnapshot(backupSite, app, progressCallback) { @@ -358,7 +358,7 @@ async function uploadAppSnapshot(backupSite, app, progressCallback) { const { stats, integrity } = await runBackupUpload(uploadConfig, progressCallback); - debug(`uploadAppSnapshot: ${app.fqdn} uploaded to ${remotePath}. ${(new Date() - startTime)/1000} seconds`); + log(`uploadAppSnapshot: ${app.fqdn} uploaded to ${remotePath}. ${(new Date() - startTime)/1000} seconds`); await backupSites.setSnapshotInfo(backupSite, app.id, { timestamp: new Date().toISOString(), manifest: app.manifest }); @@ -385,7 +385,7 @@ async function backupAppWithTag(app, backupSite, tag, options, progressCallback) const manifest = app.manifest; const remotePath = addFileExtension(backupSite, `${tag}/app_${app.fqdn}_v${manifest.version}`); - debug(`backupAppWithTag: rotating ${app.fqdn} snapshot of ${backupSite.id} to path ${remotePath}`); + log(`backupAppWithTag: rotating ${app.fqdn} snapshot of ${backupSite.id} to path ${remotePath}`); const data = { remotePath, @@ -456,7 +456,7 @@ async function uploadMailSnapshot(backupSite, progressCallback) { const { stats, integrity } = await runBackupUpload(uploadConfig, progressCallback); - debug(`uploadMailSnapshot: took ${(new Date() - startTime)/1000} seconds`); + log(`uploadMailSnapshot: took ${(new Date() - startTime)/1000} seconds`); await backupSites.setSnapshotInfo(backupSite, 'mail', { timestamp: new Date().toISOString() }); @@ -469,7 +469,7 @@ async function backupMailWithTag(backupSite, tag, options, progressCallback) { assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof progressCallback, 'function'); - debug(`backupMailWithTag: backing up mail with tag ${tag}`); + log(`backupMailWithTag: backing up mail with tag ${tag}`); const uploadStartTime = Date.now(); const uploadResult = await uploadMailSnapshot(backupSite, progressCallback); // { stats, integrity } @@ -477,7 +477,7 @@ async function backupMailWithTag(backupSite, tag, options, progressCallback) { const remotePath = addFileExtension(backupSite, `${tag}/mail_v${constants.VERSION}`); - debug(`backupMailWithTag: rotating mail snapshot of ${backupSite.id} to ${remotePath}`); + log(`backupMailWithTag: rotating mail snapshot of ${backupSite.id} to ${remotePath}`); const data = { remotePath, @@ -519,7 +519,7 @@ async function downloadMail(backupSite, remotePath, progressCallback) { const startTime = new Date(); await download(backupSite, remotePath, dataLayout, progressCallback); - debug('downloadMail: time: %s', (new Date() - startTime)/1000); + log('downloadMail: time: %s', (new Date() - startTime)/1000); } // this function is called from external process. calling process is expected to have a lock @@ -544,11 +544,11 @@ async function fullBackup(backupSiteId, options, progressCallback) { percent += step; if (!app.enableBackup) { - debug(`fullBackup: skipped backup ${app.fqdn} (${i+1}/${allApps.length}) since automatic backup disabled`); + log(`fullBackup: skipped backup ${app.fqdn} (${i+1}/${allApps.length}) since automatic backup disabled`); continue; // nothing to backup } if (!backupSites.hasContent(backupSite, app.id)) { - debug(`fullBackup: skipped backup ${app.fqdn} (${i+1}/${allApps.length}) as it is not in site contents`); + log(`fullBackup: skipped backup ${app.fqdn} (${i+1}/${allApps.length}) as it is not in site contents`); continue; } @@ -556,7 +556,7 @@ async function fullBackup(backupSiteId, options, progressCallback) { await locks.wait(`${locks.TYPE_APP_BACKUP_PREFIX}${app.id}`); const startTime = new Date(); const [appBackupError, appBackupResult] = await safe(backupAppWithTag(app, backupSite, tag, options, (progress) => progressCallback({ percent, message: progress.message }))); - debug(`fullBackup: app ${app.fqdn} backup finished. Took ${(new Date() - startTime)/1000} seconds`); + log(`fullBackup: app ${app.fqdn} backup finished. Took ${(new Date() - startTime)/1000} seconds`); await locks.release(`${locks.TYPE_APP_BACKUP_PREFIX}${app.id}`); if (appBackupError) throw appBackupError; if (appBackupResult) appBackupsMap.set(appBackupResult.id, appBackupResult.stats); // backupId can be null if in BAD_STATE and never backed up diff --git a/src/branding.js b/src/branding.js index f19ef029f..8ead49ea4 100644 --- a/src/branding.js +++ b/src/branding.js @@ -2,13 +2,13 @@ import apps from './apps.js'; import assert from 'node:assert'; import BoxError from './boxerror.js'; import constants from './constants.js'; -import debugModule from 'debug'; +import logger from './logger.js'; import eventlog from './eventlog.js'; import paths from './paths.js'; import safe from 'safetydance'; import settings from './settings.js'; -const debug = debugModule('box:branding'); +const { log, trace } = logger('branding'); async function getCloudronName() { @@ -28,7 +28,7 @@ async function setCloudronName(name, auditSource) { // mark apps using oidc addon to be reconfigured const [, installedApps] = await safe(apps.list()); - await safe(apps.configureApps(installedApps.filter((a) => !!a.manifest.addons?.oidc), { scheduleNow: true }, auditSource), { debug }); + await safe(apps.configureApps(installedApps.filter((a) => !!a.manifest.addons?.oidc), { scheduleNow: true }, auditSource), { debug: log }); await settings.set(settings.CLOUDRON_NAME_KEY, name); await eventlog.add(eventlog.ACTION_BRANDING_NAME, auditSource, { name }); diff --git a/src/community.js b/src/community.js index 8293434ed..70a961311 100644 --- a/src/community.js +++ b/src/community.js @@ -1,12 +1,12 @@ import assert from 'node:assert'; import BoxError from './boxerror.js'; -import debugModule from 'debug'; +import logger from './logger.js'; import manifestFormat from '@cloudron/manifest-format'; import promiseRetry from './promise-retry.js'; import safe from 'safetydance'; import superagent from '@cloudron/superagent'; -const debug = debugModule('box:community'); +const { log, trace } = logger('community'); const CLOUDRON_VERSIONS_FILE = 'CloudronVersions.json'; @@ -125,7 +125,7 @@ async function downloadManifest(versionsUrl) { if (!url.startsWith('https://')) throw new BoxError(BoxError.BAD_FIELD, 'versionsUrl must use https'); if (!version) throw new BoxError(BoxError.BAD_FIELD, 'version is required in versionsUrl (format: url@version)'); - debug(`downloading manifest from ${url} version ${version}`); + log(`downloading manifest from ${url} version ${version}`); const versionsRoot = await fetchVersionsRoot(url); @@ -170,7 +170,7 @@ async function getAppUpdate(app, options) { } async function downloadIcon(manifest) { - return await promiseRetry({ times: 10, interval: 5000, debug }, async function () { + return await promiseRetry({ times: 10, interval: 5000, debug: log }, async function () { const [networkError, response] = await safe(superagent.get(manifest.iconUrl) .timeout(60 * 1000) .ok(() => true)); diff --git a/src/cron.js b/src/cron.js index 95b59d1ea..5040319b3 100644 --- a/src/cron.js +++ b/src/cron.js @@ -6,7 +6,7 @@ import backupSites from './backupsites.js'; import cloudron from './cloudron.js'; import constants from './constants.js'; import { CronJob } from 'cron'; -import debugModule from 'debug'; +import logger from './logger.js'; import domains from './domains.js'; import dyndns from './dyndns.js'; import externalLdap from './externalldap.js'; @@ -24,7 +24,7 @@ import system from './system.js'; import updater from './updater.js'; import util from 'node:util'; -const debug = debugModule('box:cron'); +const { log, trace } = logger('cron'); // IMPORTANT: These patterns are together because they spin tasks which acquire a lock // If the patterns overlap all the time, then the task may not ever get a chance to run! @@ -78,7 +78,7 @@ function getCronSeed() { hour = Math.floor(24 * Math.random()); minute = Math.floor(60 * Math.random()); - debug(`getCronSeed: writing new cron seed file with ${hour}:${minute} to ${paths.CRON_SEED_FILE}`); + log(`getCronSeed: writing new cron seed file with ${hour}:${minute} to ${paths.CRON_SEED_FILE}`); safe.fs.writeFileSync(paths.CRON_SEED_FILE, `${hour}:${minute}`); } @@ -91,7 +91,7 @@ async function handleBackupScheduleChanged(site) { const tz = await cloudron.getTimeZone(); - debug(`handleBackupScheduleChanged: schedule ${site.schedule} (${tz})`); + log(`handleBackupScheduleChanged: schedule ${site.schedule} (${tz})`); if (gJobs.backups.has(site.id)) gJobs.backups.get(site.id).stop(); gJobs.backups.delete(site.id); @@ -103,7 +103,7 @@ async function handleBackupScheduleChanged(site) { onTick: async () => { const t = await backupSites.get(site.id); if (!t) return; - await safe(backupSites.startBackupTask(t, AuditSource.CRON), { debug }); + await safe(backupSites.startBackupTask(t, AuditSource.CRON), { debug: log }); }, start: true, timeZone: tz @@ -116,7 +116,7 @@ async function handleAutoupdatePatternChanged(pattern) { const tz = await cloudron.getTimeZone(); - debug(`autoupdatePatternChanged: pattern - ${pattern} (${tz})`); + log(`autoupdatePatternChanged: pattern - ${pattern} (${tz})`); if (gJobs.autoUpdater) gJobs.autoUpdater.stop(); gJobs.autoUpdater = null; @@ -125,7 +125,7 @@ async function handleAutoupdatePatternChanged(pattern) { gJobs.autoUpdater = CronJob.from({ cronTime: pattern, - onTick: async () => await safe(updater.autoUpdate(AuditSource.CRON), { debug }), + onTick: async () => await safe(updater.autoUpdate(AuditSource.CRON), { debug: log }), start: true, timeZone: tz }); @@ -134,7 +134,7 @@ async function handleAutoupdatePatternChanged(pattern) { function handleDynamicDnsChanged(enabled) { assert.strictEqual(typeof enabled, 'boolean'); - debug('Dynamic DNS setting changed to %s', enabled); + log('Dynamic DNS setting changed to %s', enabled); if (gJobs.dynamicDns) gJobs.dynamicDns.stop(); gJobs.dynamicDns = null; @@ -144,7 +144,7 @@ function handleDynamicDnsChanged(enabled) { gJobs.dynamicDns = CronJob.from({ // until we can be smarter about actual IP changes, lets ensure it every 10minutes cronTime: '00 */10 * * * *', - onTick: async () => { await safe(dyndns.refreshDns(AuditSource.CRON), { debug }); }, + onTick: async () => { await safe(dyndns.refreshDns(AuditSource.CRON), { debug: log }); }, start: true }); } @@ -159,7 +159,7 @@ async function handleExternalLdapChanged(config) { gJobs.externalLdapSyncer = CronJob.from({ cronTime: '00 00 */4 * * *', // every 4 hours - onTick: async () => await safe(externalLdap.startSyncer(AuditSource.CRON), { debug }), + onTick: async () => await safe(externalLdap.startSyncer(AuditSource.CRON), { debug: log }), start: true }); } @@ -167,42 +167,42 @@ async function handleExternalLdapChanged(config) { async function startJobs() { const { hour, minute } = getCronSeed(); - debug(`startJobs: starting cron jobs with hour ${hour} and minute ${minute}`); + log(`startJobs: starting cron jobs with hour ${hour} and minute ${minute}`); gJobs.systemChecks = CronJob.from({ cronTime: `00 ${minute} 2 * * *`, // once a day. if you change this interval, change the notification messages with correct duration - onTick: async () => await safe(system.runSystemChecks(), { debug }), + onTick: async () => await safe(system.runSystemChecks(), { debug: log }), start: true }); gJobs.mailStatusCheck = CronJob.from({ cronTime: `00 ${minute} 2 * * *`, // once a day. if you change this interval, change the notification messages with correct duration - onTick: async () => await safe(mail.checkStatus(), { debug }), + onTick: async () => await safe(mail.checkStatus(), { debug: log }), start: true }); gJobs.diskSpaceChecker = CronJob.from({ cronTime: '00 30 * * * *', // every 30 minutes. if you change this interval, change the notification messages with correct duration - onTick: async () => await safe(system.checkDiskSpace(), { debug }), + onTick: async () => await safe(system.checkDiskSpace(), { debug: log }), start: true }); // this is run separately from the update itself so that the user can disable automatic updates but can still get a notification gJobs.updateCheckerJob = CronJob.from({ cronTime: `00 ${minute} 1,5,9,13,17,21,23 * * *`, - onTick: async () => await safe(updater.checkForUpdates({ stableOnly: true }), { debug }), + onTick: async () => await safe(updater.checkForUpdates({ stableOnly: true }), { debug: log }), start: true }); gJobs.cleanupTokens = CronJob.from({ cronTime: '00 */30 * * * *', // every 30 minutes - onTick: async () => await safe(janitor.cleanupTokens(), { debug }), + onTick: async () => await safe(janitor.cleanupTokens(), { debug: log }), start: true }); gJobs.cleanupOidc = CronJob.from({ cronTime: '00 10 * * * *', // every hour ten minutes past - onTick: async () => await safe(oidcServer.cleanupExpired(), { debug }), + onTick: async () => await safe(oidcServer.cleanupExpired(), { debug: log }), start: true }); @@ -210,7 +210,7 @@ async function startJobs() { cronTime: DEFAULT_CLEANUP_BACKUPS_PATTERN, onTick: async () => { for (const backupSite of await backupSites.list()) { - await safe(backupSites.startCleanupTask(backupSite, AuditSource.CRON), { debug }); + await safe(backupSites.startCleanupTask(backupSite, AuditSource.CRON), { debug: log }); } }, start: true @@ -218,50 +218,50 @@ async function startJobs() { gJobs.cleanupEventlog = CronJob.from({ cronTime: '00 */30 * * * *', // every 30 minutes - onTick: async () => await safe(eventlog.cleanup({ creationTime: new Date(Date.now() - 90 * 60 * 24 * 60 * 1000) }), { debug }), // 90 days ago + onTick: async () => await safe(eventlog.cleanup({ creationTime: new Date(Date.now() - 90 * 60 * 24 * 60 * 1000) }), { debug: log }), // 90 days ago start: true }); gJobs.dockerVolumeCleaner = CronJob.from({ cronTime: '00 00 */12 * * *', // every 12 hours - onTick: async () => await safe(janitor.cleanupDockerVolumes(), { debug }), + onTick: async () => await safe(janitor.cleanupDockerVolumes(), { debug: log }), start: true }); gJobs.schedulerSync = CronJob.from({ cronTime: constants.TEST ? '*/10 * * * * *' : '00 */1 * * * *', // every minute - onTick: async () => await safe(scheduler.sync(), { debug }), + onTick: async () => await safe(scheduler.sync(), { debug: log }), start: true }); // randomized per Cloudron based on hourlySeed gJobs.certificateRenew = CronJob.from({ cronTime: `00 10 ${hour} * * *`, - onTick: async () => await safe(reverseProxy.startRenewCerts({}, AuditSource.CRON), { debug }), + onTick: async () => await safe(reverseProxy.startRenewCerts({}, AuditSource.CRON), { debug: log }), start: true }); gJobs.checkDomainConfigs = CronJob.from({ cronTime: `00 ${minute} 5 * * *`, // once a day - onTick: async () => await safe(domains.checkConfigs(AuditSource.CRON), { debug }), + onTick: async () => await safe(domains.checkConfigs(AuditSource.CRON), { debug: log }), start: true }); gJobs.appHealthMonitor = CronJob.from({ cronTime: '*/10 * * * * *', // every 10 seconds - onTick: async () => await safe(appHealthMonitor.run(10), { debug }), // 10 is the max run time + onTick: async () => await safe(appHealthMonitor.run(10), { debug: log }), // 10 is the max run time start: true }); gJobs.collectStats = CronJob.from({ cronTime: '*/20 * * * * *', // every 20 seconds. if you change this, change carbon config - onTick: async () => await safe(metrics.sendToGraphite(), { debug }), + onTick: async () => await safe(metrics.sendToGraphite(), { debug: log }), start: true }); gJobs.subscriptionChecker = CronJob.from({ cronTime: `00 ${minute} ${hour} * * *`, // once a day based on seed to randomize - onTick: async () => await safe(appstore.checkSubscription(), { debug }), + onTick: async () => await safe(appstore.checkSubscription(), { debug: log }), start: true }); @@ -289,7 +289,7 @@ async function stopJobs() { async function handleTimeZoneChanged(tz) { assert.strictEqual(typeof tz, 'string'); - debug('handleTimeZoneChanged: recreating all jobs'); + log('handleTimeZoneChanged: recreating all jobs'); await stopJobs(); await scheduler.deleteJobs(); // have to re-create with new tz await startJobs(); diff --git a/src/dashboard.js b/src/dashboard.js index 35f243503..6155db2a7 100644 --- a/src/dashboard.js +++ b/src/dashboard.js @@ -4,7 +4,7 @@ import assert from 'node:assert'; import BoxError from './boxerror.js'; import branding from './branding.js'; import constants from './constants.js'; -import debugModule from 'debug'; +import logger from './logger.js'; import dns from './dns.js'; import externalLdap from './externalldap.js'; import eventlog from './eventlog.js'; @@ -18,7 +18,7 @@ import system from './system.js'; import tasks from './tasks.js'; import userDirectory from './user-directory.js'; -const debug = debugModule('box:dashboard'); +const { log, trace } = logger('dashboard'); async function getLocation() { const domain = await settings.get(settings.DASHBOARD_DOMAIN_KEY); @@ -33,7 +33,7 @@ async function setLocation(subdomain, domain) { await settings.set(settings.DASHBOARD_SUBDOMAIN_KEY, subdomain); await settings.set(settings.DASHBOARD_DOMAIN_KEY, domain); - debug(`setLocation: ${domain || ''}`); + log(`setLocation: ${domain || ''}`); } async function clearLocation() { @@ -91,7 +91,7 @@ async function startPrepareLocation(domain, auditSource) { assert.strictEqual(typeof domain, 'string'); assert.strictEqual(typeof auditSource, 'object'); - debug(`prepareLocation: ${domain}`); + log(`prepareLocation: ${domain}`); if (constants.DEMO) throw new BoxError(BoxError.BAD_STATE, 'Not allowed in demo mode'); @@ -105,7 +105,7 @@ async function startPrepareLocation(domain, auditSource) { } const taskId = await tasks.add(tasks.TASK_PREPARE_DASHBOARD_LOCATION, [ constants.DASHBOARD_SUBDOMAIN, domain, auditSource ]); - safe(tasks.startTask(taskId, {}), { debug }); // background + safe(tasks.startTask(taskId, {}), { debug: log }); // background return taskId; } @@ -115,7 +115,7 @@ async function setupLocation(subdomain, domain, auditSource) { assert.strictEqual(typeof domain, 'string'); assert.strictEqual(typeof auditSource, 'object'); - debug(`setupLocation: ${domain}`); + log(`setupLocation: ${domain}`); if (constants.DEMO) throw new BoxError(BoxError.BAD_STATE, 'Not allowed in demo mode'); @@ -131,12 +131,12 @@ async function changeLocation(subdomain, domain, auditSource) { const oldLocation = await getLocation(); await setupLocation(subdomain, domain, auditSource); - debug(`setupLocation: notifying appstore and platform of domain change to ${domain}`); + log(`setupLocation: notifying appstore and platform of domain change to ${domain}`); await eventlog.add(eventlog.ACTION_DASHBOARD_DOMAIN_UPDATE, auditSource, { subdomain, domain }); - await safe(appstore.updateCloudron({ domain }), { debug }); + await safe(appstore.updateCloudron({ domain }), { debug: log }); await platform.onDashboardLocationChanged(auditSource); - await safe(reverseProxy.removeDashboardConfig(oldLocation.subdomain, oldLocation.domain), { debug }); + await safe(reverseProxy.removeDashboardConfig(oldLocation.subdomain, oldLocation.domain), { debug: log }); } const _setLocation = setLocation; diff --git a/src/database.js b/src/database.js index 2558410dd..79b487366 100644 --- a/src/database.js +++ b/src/database.js @@ -1,13 +1,13 @@ import assert from 'node:assert'; import BoxError from './boxerror.js'; import constants from './constants.js'; -import debugModule from 'debug'; +import logger from './logger.js'; import { execSync } from 'node:child_process'; import mysql from 'mysql2/promise'; import safe from 'safetydance'; import shellModule from './shell.js'; -const debug = debugModule('box:database'); +const { log, trace } = logger('database'); const shell = shellModule('database'); let gConnectionPool = null; @@ -23,9 +23,9 @@ const gDatabase = { async function uninitialize() { if (!gConnectionPool) return; - await safe(gConnectionPool.end(), { debug }); + await safe(gConnectionPool.end(), { debug: log }); gConnectionPool = null; - debug('pool closed'); + log('pool closed'); } async function query(...args) { @@ -53,7 +53,7 @@ async function transaction(queries) { connection.release(); // no await! return results; } catch (txError) { - await safe(connection.rollback(), { debug }); + await safe(connection.rollback(), { debug: log }); connection.release(); // no await! throw new BoxError(BoxError.DATABASE_ERROR, txError, { sqlCode: txError.code, sqlMessage: txError.sqlMessage || null }); } @@ -95,7 +95,7 @@ async function initialize() { // a crypto.randomUUID is 36 in length. so the value below provides for roughly 10k users await conn.query('SET SESSION group_concat_max_len = 360000'); } catch (error) { - debug(`failed to init new db connection ${connection.threadId}:`, error); // only log. we will let the app handle the exception when it calls query()/transaction() + log(`failed to init new db connection ${connection.threadId}:`, error); // only log. we will let the app handle the exception when it calls query()/transaction() } }); } @@ -129,7 +129,7 @@ async function runInTransaction(callback) { connection.release(); // no await! return result; } catch (txError) { - await safe(connection.rollback(), { debug }); + await safe(connection.rollback(), { debug: log }); connection.release(); // no await! throw new BoxError(BoxError.DATABASE_ERROR, txError, { sqlCode: txError.code, sqlMessage: txError.sqlMessage || null }); } diff --git a/src/df.js b/src/df.js index 490574554..4c93a2e73 100644 --- a/src/df.js +++ b/src/df.js @@ -1,10 +1,10 @@ import assert from 'node:assert'; import BoxError from './boxerror.js'; -import debugModule from 'debug'; +import logger from './logger.js'; import safe from 'safetydance'; import shellModule from './shell.js'; -const debug = debugModule('box:df'); +const { log, trace } = logger('df'); const shell = shellModule('df'); @@ -35,7 +35,7 @@ function parseLine(line) { async function filesystems() { const [error, output] = await safe(shell.spawn('df', ['-B1', '--output=source,fstype,size,used,avail,pcent,target'], { encoding: 'utf8', timeout: 5000 })); if (error) { - debug(`filesystems: df command failed. error: ${error}\n stdout: ${error.stdout}\n stderr: ${error.stderr}`); + log(`filesystems: df command failed. error: ${error}\n stdout: ${error.stdout}\n stderr: ${error.stderr}`); throw new BoxError(BoxError.FS_ERROR, `Error running df: ${error.message}`); } @@ -54,7 +54,7 @@ async function file(filename) { const [error, output] = await safe(shell.spawn('df', ['-B1', '--output=source,fstype,size,used,avail,pcent,target', filename], { encoding: 'utf8', timeout: 5000 })); if (error) { - debug(`file: df command failed. error: ${error}\n stdout: ${error.stdout}\n stderr: ${error.stderr}`); + log(`file: df command failed. error: ${error}\n stdout: ${error.stdout}\n stderr: ${error.stderr}`); throw new BoxError(BoxError.FS_ERROR, `Error running df: ${error.message}`); } diff --git a/src/directoryserver.js b/src/directoryserver.js index 7f0cc6188..13a13f2a0 100644 --- a/src/directoryserver.js +++ b/src/directoryserver.js @@ -2,7 +2,7 @@ import assert from 'node:assert'; import AuditSource from './auditsource.js'; import BoxError from './boxerror.js'; import constants from './constants.js'; -import debugModule from 'debug'; +import logger from './logger.js'; import eventlog from './eventlog.js'; import ipaddr from './ipaddr.js'; import groups from './groups.js'; @@ -16,7 +16,7 @@ import shellModule from './shell.js'; import users from './users.js'; import util from 'node:util'; -const debug = debugModule('box:directoryserver'); +const { log, trace } = logger('directoryserver'); const shell = shellModule('directoryserver'); @@ -57,7 +57,7 @@ async function validateConfig(config) { } async function authorize(req, res, next) { - debug('authorize: ', req.connection.ldap.bindDN.toString()); + log('authorize: ', req.connection.ldap.bindDN.toString()); // this is for connection attempts without previous bind if (req.connection.ldap.bindDN.equals('cn=anonymous')) return next(new ldap.InsufficientAccessRightsError()); @@ -69,7 +69,7 @@ async function authorize(req, res, next) { } async function maybeRootDSE(req, res, next) { - debug(`maybeRootDSE: requested with scope:${req.scope} dn:${req.dn.toString()}`); + log(`maybeRootDSE: requested with scope:${req.scope} dn:${req.dn.toString()}`); if (req.scope !== 'base') return next(new ldap.NoSuchObjectError()); // per the spec, rootDSE search require base scope if (!req.dn || req.dn.toString() !== '') return next(new ldap.NoSuchObjectError()); @@ -126,7 +126,7 @@ async function userAuth(req, res, next) { async function stop() { if (!gServer) return; - debug('stopping server'); + log('stopping server'); await util.promisify(gServer.close.bind(gServer))(); gServer = null; @@ -192,7 +192,7 @@ function finalSend(results, req, res, next) { // Will attach req.user if successful async function userSearch(req, res, next) { - debug('user search: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id); + log('user search: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id); const [error, allUsers] = await safe(users.list()); if (error) return next(new ldap.OperationsError(error.message)); @@ -248,7 +248,7 @@ async function userSearch(req, res, next) { } async function groupSearch(req, res, next) { - debug('group search: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id); + log('group search: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id); const [error, allUsers] = await safe(users.list()); if (error) return next(new ldap.OperationsError(error.message)); @@ -292,10 +292,10 @@ async function start() { const logger = { trace: NOOP, debug: NOOP, - info: debug, - warn: debug, - error: debug, - fatal: debug + info: log, + warn: log, + error: log, + fatal: log }; gCertificate = await reverseProxy.getDirectoryServerCertificate(); @@ -307,11 +307,11 @@ async function start() { }); gServer.on('error', function (error) { - debug('server startup error: %o', error); + log('server startup error: %o', error); }); gServer.bind('ou=system,dc=cloudron', async function(req, res, next) { - debug('system bind: %s (from %s)', req.dn.toString(), req.connection.ldap.id); + log('system bind: %s (from %s)', req.dn.toString(), req.connection.ldap.id); const config = await getConfig(); @@ -340,11 +340,11 @@ async function start() { // just log that an attempt was made to unknown route, this helps a lot during app packaging gServer.use(function(req, res, next) { - debug('not handled: dn %s, scope %s, filter %s (from %s)', req.dn ? req.dn.toString() : '-', req.scope, req.filter ? req.filter.toString() : '-', req.connection.ldap.id); + log('not handled: dn %s, scope %s, filter %s (from %s)', req.dn ? req.dn.toString() : '-', req.scope, req.filter ? req.filter.toString() : '-', req.connection.ldap.id); return next(); }); - debug(`starting server on port ${constants.USER_DIRECTORY_LDAPS_PORT}`); + log(`starting server on port ${constants.USER_DIRECTORY_LDAPS_PORT}`); await util.promisify(gServer.listen.bind(gServer))(constants.USER_DIRECTORY_LDAPS_PORT, '::'); } @@ -397,11 +397,11 @@ async function checkCertificate() { const certificate = await reverseProxy.getDirectoryServerCertificate(); if (certificate.cert === gCertificate.cert) { - debug('checkCertificate: certificate has not changed'); + log('checkCertificate: certificate has not changed'); return; } - debug('checkCertificate: certificate changed. restarting'); + log('checkCertificate: certificate changed. restarting'); await stop(); await start(); } diff --git a/src/dns.js b/src/dns.js index a89e5d995..191e2f998 100644 --- a/src/dns.js +++ b/src/dns.js @@ -3,7 +3,7 @@ import assert from 'node:assert'; import BoxError from './boxerror.js'; import constants from './constants.js'; import dashboard from './dashboard.js'; -import debugModule from 'debug'; +import logger from './logger.js'; import domains from './domains.js'; import ipaddr from './ipaddr.js'; import mail from './mail.js'; @@ -36,7 +36,7 @@ import dnsOvh from './dns/ovh.js'; import dnsPorkbun from './dns/porkbun.js'; import dnsWildcard from './dns/wildcard.js'; -const debug = debugModule('box:dns'); +const { log, trace } = logger('dns'); const DNS_PROVIDERS = { @@ -140,7 +140,7 @@ async function upsertDnsRecords(subdomain, domain, type, values) { assert.strictEqual(typeof type, 'string'); assert(Array.isArray(values)); - debug(`upsertDnsRecords: subdomain:${subdomain} domain:${domain} type:${type} values:${JSON.stringify(values)}`); + log(`upsertDnsRecords: subdomain:${subdomain} domain:${domain} type:${type} values:${JSON.stringify(values)}`); const domainObject = await domains.get(domain); await api(domainObject.provider).upsert(domainObject, subdomain, type, values); @@ -152,7 +152,7 @@ async function removeDnsRecords(subdomain, domain, type, values) { assert.strictEqual(typeof type, 'string'); assert(Array.isArray(values)); - debug(`removeDnsRecords: subdomain:${subdomain} domain:${domain} type:${type} values:${JSON.stringify(values)}`); + log(`removeDnsRecords: subdomain:${subdomain} domain:${domain} type:${type} values:${JSON.stringify(values)}`); const domainObject = await domains.get(domain); const [error] = await safe(api(domainObject.provider).del(domainObject, subdomain, type, values)); @@ -220,7 +220,7 @@ async function registerLocation(location, options, recordType, recordValue) { const [getError, values] = await safe(getDnsRecords(location.subdomain, location.domain, recordType)); if (getError) { const retryable = getError.reason !== BoxError.ACCESS_DENIED && getError.reason !== BoxError.NOT_FOUND; // NOT_FOUND is when zone is not found - debug(`registerLocation: Get error. retryable: ${retryable}. ${getError.message}`); + log(`registerLocation: Get error. retryable: ${retryable}. ${getError.message}`); throw new BoxError(getError.reason, `${location.domain}: ${getError.message}`, { domain: location, retryable }); } @@ -232,7 +232,7 @@ async function registerLocation(location, options, recordType, recordValue) { const [upsertError] = await safe(upsertDnsRecords(location.subdomain, location.domain, recordType, [ recordValue ])); if (upsertError) { const retryable = upsertError.reason === BoxError.BUSY || upsertError.reason === BoxError.EXTERNAL_ERROR; - debug(`registerLocation: Upsert error. retryable: ${retryable}. ${upsertError.message}`); + log(`registerLocation: Upsert error. retryable: ${retryable}. ${upsertError.message}`); throw new BoxError(BoxError.EXTERNAL_ERROR, `${location.domain}: ${getError.message}`, { domain: location, retryable }); } } @@ -242,7 +242,7 @@ async function registerLocations(locations, options, progressCallback) { assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof progressCallback, 'function'); - debug(`registerLocations: Will register ${JSON.stringify(locations)} with options ${JSON.stringify(options)}`); + log(`registerLocations: Will register ${JSON.stringify(locations)} with options ${JSON.stringify(options)}`); const ipv4 = await network.getIPv4(); const ipv6 = await network.getIPv6(); @@ -250,12 +250,12 @@ async function registerLocations(locations, options, progressCallback) { for (const location of locations) { progressCallback({ message: `Registering location ${fqdn(location.subdomain, location.domain)}` }); - await promiseRetry({ times: 200, interval: 5000, debug, retry: (error) => error.retryable }, async function () { + await promiseRetry({ times: 200, interval: 5000, debug: log, retry: (error) => error.retryable }, async function () { // cname records cannot co-exist with other records const [getError, values] = await safe(getDnsRecords(location.subdomain, location.domain, 'CNAME')); if (!getError && values.length === 1) { if (!options.overwriteDns) throw new BoxError(BoxError.ALREADY_EXISTS, 'DNS CNAME record already exists', { domain: location, retryable: false }); - debug(`registerLocations: removing CNAME record of ${fqdn(location.subdomain, location.domain)}`); + log(`registerLocations: removing CNAME record of ${fqdn(location.subdomain, location.domain)}`); await removeDnsRecords(location.subdomain, location.domain, 'CNAME', values); } @@ -263,14 +263,14 @@ async function registerLocations(locations, options, progressCallback) { await registerLocation(location, options, 'A', ipv4); } else { const [aError, aValues] = await safe(getDnsRecords(location.subdomain, location.domain, 'A')); - if (!aError && aValues.length) await safe(removeDnsRecords(location.subdomain, location.domain, 'A', aValues), { debug }); + if (!aError && aValues.length) await safe(removeDnsRecords(location.subdomain, location.domain, 'A', aValues), { debug: log }); } if (ipv6) { await registerLocation(location, options, 'AAAA', ipv6); } else { const [aaaaError, aaaaValues] = await safe(getDnsRecords(location.subdomain, location.domain, 'AAAA')); - if (!aaaaError && aaaaValues.length) await safe(removeDnsRecords(location.subdomain, location.domain, 'AAAA', aaaaValues), { debug }); + if (!aaaaError && aaaaValues.length) await safe(removeDnsRecords(location.subdomain, location.domain, 'AAAA', aaaaValues), { debug: log }); } }); } @@ -285,7 +285,7 @@ async function unregisterLocation(location, recordType, recordValue) { if (!error || error.reason === BoxError.NOT_FOUND) return; const retryable = error.reason === BoxError.BUSY || error.reason === BoxError.EXTERNAL_ERROR; - debug(`unregisterLocation: Error unregistering location ${recordType}. retryable: ${retryable}. ${error.message}`); + log(`unregisterLocation: Error unregistering location ${recordType}. retryable: ${retryable}. ${error.message}`); throw new BoxError(BoxError.EXTERNAL_ERROR, `${location.domain}: ${error.message}`, { domain: location, retryable }); } @@ -300,7 +300,7 @@ async function unregisterLocations(locations, progressCallback) { for (const location of locations) { progressCallback({ message: `Unregistering location: ${location.subdomain ? (location.subdomain + '.') : ''}${location.domain}` }); - await promiseRetry({ times: 30, interval: 5000, debug, retry: (error) => error.retryable }, async function () { + await promiseRetry({ times: 30, interval: 5000, debug: log, retry: (error) => error.retryable }, async function () { if (ipv4) await unregisterLocation(location, 'A', ipv4); if (ipv6) await unregisterLocation(location, 'AAAA', ipv6); }); @@ -364,7 +364,7 @@ async function startSyncDnsRecords(options) { assert.strictEqual(typeof options, 'object'); const taskId = await tasks.add(tasks.TASK_SYNC_DNS_RECORDS, [ options ]); - safe(tasks.startTask(taskId, {}), { debug }); // background + safe(tasks.startTask(taskId, {}), { debug: log }); // background return taskId; } diff --git a/src/dns/bunny.js b/src/dns/bunny.js index c449041b5..1e48eae41 100644 --- a/src/dns/bunny.js +++ b/src/dns/bunny.js @@ -1,14 +1,14 @@ import assert from 'node:assert'; import BoxError from '../boxerror.js'; import constants from '../constants.js'; -import debugModule from 'debug'; +import logger from '../logger.js'; import dig from '../dig.js'; import dns from '../dns.js'; import safe from 'safetydance'; import superagent from '@cloudron/superagent'; import waitForDns from './waitfordns.js'; -const debug = debugModule('box:dns/bunny'); +const { log, trace } = logger('dns/bunny'); const BUNNY_API = 'https://api.bunny.net'; @@ -61,7 +61,7 @@ async function getDnsRecords(domainConfig, zoneName, name, type) { assert.strictEqual(typeof name, 'string'); assert.strictEqual(typeof type, 'string'); - debug(`get: ${name} in zone ${zoneName} of type ${type}`); + log(`get: ${name} in zone ${zoneName} of type ${type}`); const zoneId = await getZoneId(domainConfig, zoneName); const [error, response] = await safe(superagent.get(`${BUNNY_API}/dnszone/${zoneId}`) @@ -87,7 +87,7 @@ async function upsert(domainObject, location, type, values) { zoneName = domainObject.zoneName, name = dns.getName(domainObject, location, type) || ''; - debug(`upsert: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`); + log(`upsert: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`); const zoneId = await getZoneId(domainConfig, zoneName); const records = await getDnsRecords(domainConfig, zoneName, name, type); @@ -148,10 +148,10 @@ async function upsert(domainObject, location, type, values) { .timeout(30 * 1000) .ok(() => true)); - if (error) debug(`upsert: error removing record ${records[j].id}: ${error.message}`); + if (error) log(`upsert: error removing record ${records[j].id}: ${error.message}`); } - debug('upsert: completed with recordIds:%j', recordIds); + log('upsert: completed with recordIds:%j', recordIds); } async function get(domainObject, location, type) { @@ -177,7 +177,7 @@ async function del(domainObject, location, type, values) { zoneName = domainObject.zoneName, name = dns.getName(domainObject, location, type) || ''; - debug(`del: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`); + log(`del: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`); const zoneId = await getZoneId(domainConfig, zoneName); const records = await getDnsRecords(domainConfig, zoneName, name, type); @@ -232,17 +232,17 @@ async function verifyDomainConfig(domainObject) { if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers'); if (!nameservers.every(function (n) { return n.toLowerCase().indexOf('.bunny.net') !== -1; })) { - debug('verifyDomainConfig: %j does not contain Bunny NS', nameservers); + log('verifyDomainConfig: %j does not contain Bunny NS', nameservers); if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Bunny'); } const location = 'cloudrontestdns'; await upsert(domainObject, location, 'A', [ ip ]); - debug('verifyDomainConfig: Test A record added'); + log('verifyDomainConfig: Test A record added'); await del(domainObject, location, 'A', [ ip ]); - debug('verifyDomainConfig: Test A record removed again'); + log('verifyDomainConfig: Test A record removed again'); return credentials; } diff --git a/src/dns/cloudflare.js b/src/dns/cloudflare.js index 10548509c..86f7fb4e1 100644 --- a/src/dns/cloudflare.js +++ b/src/dns/cloudflare.js @@ -1,7 +1,7 @@ import assert from 'node:assert'; import BoxError from '../boxerror.js'; import constants from '../constants.js'; -import debugModule from 'debug'; +import logger from '../logger.js'; import dig from '../dig.js'; import dns from '../dns.js'; import safe from 'safetydance'; @@ -9,7 +9,7 @@ import superagent from '@cloudron/superagent'; import waitForDns from './waitfordns.js'; import _ from '../underscore.js'; -const debug = debugModule('box:dns/cloudflare'); +const { log, trace } = logger('dns/cloudflare'); // we are using latest v4 stable API https://api.cloudflare.com/#getting-started-endpoints @@ -107,7 +107,7 @@ async function upsert(domainObject, location, type, values) { zoneName = domainObject.zoneName, fqdn = dns.fqdn(location, domainObject.domain); - debug('upsert: %s for zone %s of type %s with values %j', fqdn, zoneName, type, values); + log('upsert: %s for zone %s of type %s with values %j', fqdn, zoneName, type, values); const zone = await getZoneByName(domainConfig, zoneName); const zoneId = zone.id; @@ -138,7 +138,7 @@ async function upsert(domainObject, location, type, values) { data.proxied = !!domainConfig.defaultProxyStatus; // note that cloudflare will error if proxied is set for wrong record type or IP. only set at install time } - debug(`upsert: Adding new record fqdn: ${fqdn}, zoneName: ${zoneName} proxied: ${data.proxied}`); + log(`upsert: Adding new record fqdn: ${fqdn}, zoneName: ${zoneName} proxied: ${data.proxied}`); const [error, response] = await safe(createRequest('POST', `${CLOUDFLARE_ENDPOINT}/zones/${zoneId}/dns_records`, domainConfig) .send(data)); @@ -147,7 +147,7 @@ async function upsert(domainObject, location, type, values) { } else { // replace existing record data.proxied = records[i].proxied; // preserve proxied parameter - debug(`upsert: Updating existing record fqdn: ${fqdn}, zoneName: ${zoneName} proxied: ${data.proxied}`); + log(`upsert: Updating existing record fqdn: ${fqdn}, zoneName: ${zoneName} proxied: ${data.proxied}`); const [error, response] = await safe(createRequest('PUT', `${CLOUDFLARE_ENDPOINT}/zones/${zoneId}/dns_records/${records[i].id}`, domainConfig) .send(data)); @@ -160,7 +160,7 @@ async function upsert(domainObject, location, type, values) { for (let j = values.length + 1; j < records.length; j++) { const [error] = await safe(createRequest('DELETE', `${CLOUDFLARE_ENDPOINT}/zones/${zoneId}/dns_records/${records[j].id}`, domainConfig)); - if (error) debug(`upsert: error removing record ${records[j].id}: ${error.message}`); + if (error) log(`upsert: error removing record ${records[j].id}: ${error.message}`); } } @@ -195,7 +195,7 @@ async function del(domainObject, location, type, values) { if (result.length === 0) return; const tmp = result.filter(function (record) { return values.some(function (value) { return value === record.content; }); }); - debug('del: %j', tmp); + log('del: %j', tmp); if (tmp.length === 0) return; @@ -217,7 +217,7 @@ async function wait(domainObject, subdomain, type, value, options) { zoneName = domainObject.zoneName, fqdn = dns.fqdn(subdomain, domainObject.domain); - debug('wait: %s for zone %s of type %s', fqdn, zoneName, type); + log('wait: %s for zone %s of type %s', fqdn, zoneName, type); const zone = await getZoneByName(domainConfig, zoneName); const zoneId = zone.id; @@ -227,7 +227,7 @@ async function wait(domainObject, subdomain, type, value, options) { if (!dnsRecords[0].proxied) return await waitForDns(fqdn, domainObject.zoneName, type, value, options); - debug('wait: skipping wait of proxied domain'); + log('wait: skipping wait of proxied domain'); // maybe we can check for dns to be cloudflare IPs? https://api.cloudflare.com/#cloudflare-ips-cloudflare-ip-details } @@ -268,17 +268,17 @@ async function verifyDomainConfig(domainObject) { const zone = await getZoneByName(domainConfig, zoneName); if (!_.isEqual(zone.name_servers.sort(), nameservers.sort())) { - debug('verifyDomainConfig: %j and %j do not match', nameservers, zone.name_servers); + log('verifyDomainConfig: %j and %j do not match', nameservers, zone.name_servers); if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Cloudflare'); } const location = 'cloudrontestdns'; await upsert(domainObject, location, 'A', [ ip ]); - debug('verifyDomainConfig: Test A record added'); + log('verifyDomainConfig: Test A record added'); await del(domainObject, location, 'A', [ ip ]); - debug('verifyDomainConfig: Test A record removed again'); + log('verifyDomainConfig: Test A record removed again'); return sanitizedConfig; } diff --git a/src/dns/desec.js b/src/dns/desec.js index 8c16c48e0..e70770888 100644 --- a/src/dns/desec.js +++ b/src/dns/desec.js @@ -1,7 +1,7 @@ import assert from 'node:assert'; import constants from '../constants.js'; import BoxError from '../boxerror.js'; -import debugModule from 'debug'; +import logger from '../logger.js'; import dig from '../dig.js'; import dns from '../dns.js'; import safe from 'safetydance'; @@ -9,7 +9,7 @@ import timers from 'timers/promises'; import superagent from '@cloudron/superagent'; import waitForDns from './waitfordns.js'; -const debug = debugModule('box:dns/desec'); +const { log, trace } = logger('dns/desec'); const DESEC_ENDPOINT = 'https://desec.io/api/v1'; @@ -143,17 +143,17 @@ async function verifyDomainConfig(domainObject) { if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers'); if (!nameservers.every(function (n) { return n.toLowerCase().indexOf('.desec.') !== -1; })) { - debug('verifyDomainConfig: %j does not contains deSEC NS', nameservers); + log('verifyDomainConfig: %j does not contains deSEC NS', nameservers); if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to deSEC'); } const location = 'cloudrontestdns'; await upsert(domainObject, location, 'A', [ ip ]); - debug('verifyDomainConfig: Test A record added'); + log('verifyDomainConfig: Test A record added'); await del(domainObject, location, 'A', [ ip ]); - debug('verifyDomainConfig: Test A record removed again'); + log('verifyDomainConfig: Test A record removed again'); return credentials; } diff --git a/src/dns/digitalocean.js b/src/dns/digitalocean.js index 099e18207..e31a847a8 100644 --- a/src/dns/digitalocean.js +++ b/src/dns/digitalocean.js @@ -1,14 +1,14 @@ import assert from 'node:assert'; import BoxError from '../boxerror.js'; import constants from '../constants.js'; -import debugModule from 'debug'; +import logger from '../logger.js'; import dig from '../dig.js'; import dns from '../dns.js'; import safe from 'safetydance'; import superagent from '@cloudron/superagent'; import waitForDns from './waitfordns.js'; -const debug = debugModule('box:dns/digitalocean'); +const { log, trace } = logger('dns/digitalocean'); const DIGITALOCEAN_ENDPOINT = 'https://api.digitalocean.com'; @@ -34,7 +34,7 @@ async function getZoneRecords(domainConfig, zoneName, name, type) { let nextPage = null, matchingRecords = []; - debug(`getInternal: getting dns records of ${zoneName} with ${name} and type ${type}`); + log(`getInternal: getting dns records of ${zoneName} with ${name} and type ${type}`); do { const url = nextPage ? nextPage : DIGITALOCEAN_ENDPOINT + '/v2/domains/' + zoneName + '/records'; @@ -70,7 +70,7 @@ async function upsert(domainObject, location, type, values) { zoneName = domainObject.zoneName, name = dns.getName(domainObject, location, type) || '@'; - debug('upsert: %s for zone %s of type %s with values %j', name, zoneName, type, values); + log('upsert: %s for zone %s of type %s with values %j', name, zoneName, type, values); const records = await getZoneRecords(domainConfig, zoneName, name, type); @@ -136,10 +136,10 @@ async function upsert(domainObject, location, type, values) { .retry(5) .ok(() => true)); - if (error) debug(`upsert: error removing record ${records[j].id}: ${error.message}`); + if (error) log(`upsert: error removing record ${records[j].id}: ${error.message}`); } - debug('upsert: completed with recordIds:%j', recordIds); + log('upsert: completed with recordIds:%j', recordIds); } async function get(domainObject, location, type) { @@ -229,17 +229,17 @@ async function verifyDomainConfig(domainObject) { if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers'); if (nameservers.map(function (n) { return n.toLowerCase(); }).indexOf('ns1.digitalocean.com') === -1) { - debug('verifyDomainConfig: %j does not contains DO NS', nameservers); + log('verifyDomainConfig: %j does not contains DO NS', nameservers); if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to DigitalOcean'); } const location = 'cloudrontestdns'; await upsert(domainObject, location, 'A', [ ip ]); - debug('verifyDomainConfig: Test A record added'); + log('verifyDomainConfig: Test A record added'); await del(domainObject, location, 'A', [ ip ]); - debug('verifyDomainConfig: Test A record removed again'); + log('verifyDomainConfig: Test A record removed again'); return credentials; } diff --git a/src/dns/dnsimple.js b/src/dns/dnsimple.js index 6ed2376ad..90135cadc 100644 --- a/src/dns/dnsimple.js +++ b/src/dns/dnsimple.js @@ -1,14 +1,14 @@ import assert from 'node:assert'; import BoxError from '../boxerror.js'; import constants from '../constants.js'; -import debugModule from 'debug'; +import logger from '../logger.js'; import dig from '../dig.js'; import dns from '../dns.js'; import safe from 'safetydance'; import superagent from '@cloudron/superagent'; import waitForDns from './waitfordns.js'; -const debug = debugModule('box:dns/dnsimple'); +const { log, trace } = logger('dns/dnsimple'); const DNSIMPLE_API = 'https://api.dnsimple.com/v2'; @@ -70,7 +70,7 @@ async function getDnsRecords(domainConfig, zoneName, name, type) { assert.strictEqual(typeof name, 'string'); assert.strictEqual(typeof type, 'string'); - debug(`get: ${name} in zone ${zoneName} of type ${type}`); + log(`get: ${name} in zone ${zoneName} of type ${type}`); const { accountId, zoneId } = await getZone(domainConfig, zoneName); const [error, response] = await safe(superagent.get(`${DNSIMPLE_API}/${accountId}/zones/${zoneId}/records?name=${name}&type=${type}`) @@ -96,7 +96,7 @@ async function upsert(domainObject, location, type, values) { zoneName = domainObject.zoneName, name = dns.getName(domainObject, location, type) || ''; - debug(`upsert: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`); + log(`upsert: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`); const { accountId, zoneId } = await getZone(domainConfig, zoneName); const records = await getDnsRecords(domainConfig, zoneName, name, type); @@ -157,10 +157,10 @@ async function upsert(domainObject, location, type, values) { .timeout(30 * 1000) .ok(() => true)); - if (error) debug(`upsert: error removing record ${records[j].id}: ${error.message}`); + if (error) log(`upsert: error removing record ${records[j].id}: ${error.message}`); } - debug('upsert: completed with recordIds:%j', recordIds); + log('upsert: completed with recordIds:%j', recordIds); } async function get(domainObject, location, type) { @@ -186,7 +186,7 @@ async function del(domainObject, location, type, values) { zoneName = domainObject.zoneName, name = dns.getName(domainObject, location, type) || ''; - debug(`del: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`); + log(`del: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`); const { accountId, zoneId } = await getZone(domainConfig, zoneName); const records = await getDnsRecords(domainConfig, zoneName, name, type); @@ -241,17 +241,17 @@ async function verifyDomainConfig(domainObject) { if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers'); if (!nameservers.every(function (n) { return n.toLowerCase().indexOf('dnsimple') !== -1; })) { // can be dnsimple.com or dnsimple-edge.org - debug('verifyDomainConfig: %j does not contain dnsimple NS', nameservers); + log('verifyDomainConfig: %j does not contain dnsimple NS', nameservers); if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to dnsimple'); } const location = 'cloudrontestdns'; await upsert(domainObject, location, 'A', [ ip ]); - debug('verifyDomainConfig: Test A record added'); + log('verifyDomainConfig: Test A record added'); await del(domainObject, location, 'A', [ ip ]); - debug('verifyDomainConfig: Test A record removed again'); + log('verifyDomainConfig: Test A record removed again'); return credentials; } diff --git a/src/dns/gandi.js b/src/dns/gandi.js index ed5edab20..fed992c79 100644 --- a/src/dns/gandi.js +++ b/src/dns/gandi.js @@ -1,14 +1,14 @@ import assert from 'node:assert'; import BoxError from '../boxerror.js'; import constants from '../constants.js'; -import debugModule from 'debug'; +import logger from '../logger.js'; import dig from '../dig.js'; import dns from '../dns.js'; import safe from 'safetydance'; import superagent from '@cloudron/superagent'; import waitForDns from './waitfordns.js'; -const debug = debugModule('box:dns/gandi'); +const { log, trace } = logger('dns/gandi'); const GANDI_API = 'https://dns.api.gandi.net/api/v5'; @@ -54,7 +54,7 @@ async function upsert(domainObject, location, type, values) { zoneName = domainObject.zoneName, name = dns.getName(domainObject, location, type) || '@'; - debug(`upsert: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`); + log(`upsert: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`); const data = { 'rrset_ttl': 300, // this is the minimum allowed @@ -79,7 +79,7 @@ async function get(domainObject, location, type) { zoneName = domainObject.zoneName, name = dns.getName(domainObject, location, type) || '@'; - debug(`get: ${name} in zone ${zoneName} of type ${type}`); + log(`get: ${name} in zone ${zoneName} of type ${type}`); const [error, response] = await safe(createRequest('GET', `${GANDI_API}/domains/${zoneName}/records/${name}/${type}`, domainConfig)); @@ -101,7 +101,7 @@ async function del(domainObject, location, type, values) { zoneName = domainObject.zoneName, name = dns.getName(domainObject, location, type) || '@'; - debug(`del: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`); + log(`del: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`); const [error, response] = await safe(createRequest('DELETE', `${GANDI_API}/domains/${zoneName}/records/${name}/${type}`, domainConfig)); @@ -148,17 +148,17 @@ async function verifyDomainConfig(domainObject) { if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers'); if (!nameservers.every(function (n) { return n.toLowerCase().indexOf('.gandi.net') !== -1; })) { - debug('verifyDomainConfig: %j does not contain Gandi NS', nameservers); + log('verifyDomainConfig: %j does not contain Gandi NS', nameservers); if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Gandi'); } const location = 'cloudrontestdns'; await upsert(domainObject, location, 'A', [ ip ]); - debug('verifyDomainConfig: Test A record added'); + log('verifyDomainConfig: Test A record added'); await del(domainObject, location, 'A', [ ip ]); - debug('verifyDomainConfig: Test A record removed again'); + log('verifyDomainConfig: Test A record removed again'); return credentials; } diff --git a/src/dns/gcdns.js b/src/dns/gcdns.js index eaa22502e..60ed229b0 100644 --- a/src/dns/gcdns.js +++ b/src/dns/gcdns.js @@ -1,7 +1,7 @@ import assert from 'node:assert'; import BoxError from '../boxerror.js'; import constants from '../constants.js'; -import debugModule from 'debug'; +import logger from '../logger.js'; import dig from '../dig.js'; import dns from '../dns.js'; import { DNS as GCDNS } from '@google-cloud/dns'; @@ -9,7 +9,7 @@ import safe from 'safetydance'; import waitForDns from './waitfordns.js'; import _ from '../underscore.js'; -const debug = debugModule('box:dns/gcdns'); +const { log, trace } = logger('dns/gcdns'); function removePrivateFields(domainObject) { @@ -66,7 +66,7 @@ async function upsert(domainObject, location, type, values) { zoneName = domainObject.zoneName, fqdn = dns.fqdn(location, domainObject.domain); - debug('add: %s for zone %s of type %s with values %j', fqdn, zoneName, type, values); + log('add: %s for zone %s of type %s with values %j', fqdn, zoneName, type, values); const zone = await getZoneByName(getDnsCredentials(domainConfig), zoneName); @@ -170,16 +170,16 @@ async function verifyDomainConfig(domainObject) { const definedNS = zone.metadata.nameServers.map(function(r) { return r.replace(/\.$/, ''); }); if (!_.isEqual(definedNS.sort(), nameservers.sort())) { - debug('verifyDomainConfig: %j and %j do not match', nameservers, definedNS); + log('verifyDomainConfig: %j and %j do not match', nameservers, definedNS); if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Google Cloud DNS'); } const location = 'cloudrontestdns'; await upsert(domainObject, location, 'A', [ ip ]); - debug('verifyDomainConfig: Test A record added'); + log('verifyDomainConfig: Test A record added'); await del(domainObject, location, 'A', [ ip ]); - debug('verifyDomainConfig: Test A record removed again'); + log('verifyDomainConfig: Test A record removed again'); return credentials; } diff --git a/src/dns/godaddy.js b/src/dns/godaddy.js index 9c1adbcf7..5337b3ed7 100644 --- a/src/dns/godaddy.js +++ b/src/dns/godaddy.js @@ -1,14 +1,14 @@ import assert from 'node:assert'; import BoxError from '../boxerror.js'; import constants from '../constants.js'; -import debugModule from 'debug'; +import logger from '../logger.js'; import dig from '../dig.js'; import dns from '../dns.js'; import safe from 'safetydance'; import superagent from '@cloudron/superagent'; import waitForDns from './waitfordns.js'; -const debug = debugModule('box:dns/godaddy'); +const { log, trace } = logger('dns/godaddy'); // const GODADDY_API_OTE = 'https://api.ote-godaddy.com/v1/domains'; @@ -37,7 +37,7 @@ async function upsert(domainObject, location, type, values) { zoneName = domainObject.zoneName, name = dns.getName(domainObject, location, type) || '@'; - debug(`upsert: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`); + log(`upsert: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`); const records = []; for (const value of values) { @@ -75,7 +75,7 @@ async function get(domainObject, location, type) { zoneName = domainObject.zoneName, name = dns.getName(domainObject, location, type) || '@'; - debug(`get: ${name} in zone ${zoneName} of type ${type}`); + log(`get: ${name} in zone ${zoneName} of type ${type}`); const [error, response] = await safe(superagent.get(`${GODADDY_API}/${zoneName}/records/${type}/${name}`) .set('Authorization', `sso-key ${domainConfig.apiKey}:${domainConfig.apiSecret}`) @@ -114,7 +114,7 @@ async function del(domainObject, location, type, values) { zoneName = domainObject.zoneName, name = dns.getName(domainObject, location, type) || '@'; - debug(`del: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`); + log(`del: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`); const result = await get(domainObject, location, type); if (result.length === 0) return; @@ -171,17 +171,17 @@ async function verifyDomainConfig(domainObject) { if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers'); if (!nameservers.every(function (n) { return n.toLowerCase().indexOf('.domaincontrol.com') !== -1 || n.toLowerCase().indexOf('.secureserver.net') !== -1; })) { - debug('verifyDomainConfig: %j does not contain GoDaddy NS', nameservers); + log('verifyDomainConfig: %j does not contain GoDaddy NS', nameservers); if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to GoDaddy'); } const location = 'cloudrontestdns'; await upsert(domainObject, location, 'A', [ ip ]); - debug('verifyDomainConfig: Test A record added'); + log('verifyDomainConfig: Test A record added'); await del(domainObject, location, 'A', [ ip ]); - debug('verifyDomainConfig: Test A record removed again'); + log('verifyDomainConfig: Test A record removed again'); return credentials; } diff --git a/src/dns/hetzner.js b/src/dns/hetzner.js index fab02d380..141201c0e 100644 --- a/src/dns/hetzner.js +++ b/src/dns/hetzner.js @@ -1,14 +1,14 @@ import assert from 'node:assert'; import BoxError from '../boxerror.js'; import constants from '../constants.js'; -import debugModule from 'debug'; +import logger from '../logger.js'; import dig from '../dig.js'; import dns from '../dns.js'; import safe from 'safetydance'; import superagent from '@cloudron/superagent'; import waitForDns from './waitfordns.js'; -const debug = debugModule('box:dns/hetzner'); +const { log, trace } = logger('dns/hetzner'); const ENDPOINT = 'https://dns.hetzner.com/api/v1'; @@ -55,7 +55,7 @@ async function getZoneRecords(domainConfig, zone, name, type) { let page = 1, matchingRecords = []; - debug(`getZoneRecords: getting dns records of ${zone.name} with ${name} and type ${type}`); + log(`getZoneRecords: getting dns records of ${zone.name} with ${name} and type ${type}`); const perPage = 50; @@ -94,7 +94,7 @@ async function upsert(domainObject, location, type, values) { zoneName = domainObject.zoneName, name = dns.getName(domainObject, location, type) || '@'; - debug(`upsert: ${name} for zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`); + log(`upsert: ${name} for zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`); const zone = await getZone(domainConfig, zoneName); const records = await getZoneRecords(domainConfig, zone, name, type); @@ -147,10 +147,10 @@ async function upsert(domainObject, location, type, values) { .retry(5) .ok(() => true)); - if (error) debug(`upsert: error removing record ${records[j].id}: ${error.message}`); + if (error) log(`upsert: error removing record ${records[j].id}: ${error.message}`); } - debug('upsert: completed'); + log('upsert: completed'); } async function get(domainObject, location, type) { @@ -235,17 +235,17 @@ async function verifyDomainConfig(domainObject) { // https://docs.hetzner.com/dns-console/dns/general/dns-overview#the-hetzner-online-name-servers-are if (!nameservers.every(function (n) { return n.toLowerCase().search(/hetzner|your-server|second-ns/) !== -1; })) { - debug('verifyDomainConfig: %j does not contain Hetzner NS', nameservers); + log('verifyDomainConfig: %j does not contain Hetzner NS', nameservers); if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Hetzner'); } const location = 'cloudrontestdns'; await upsert(domainObject, location, 'A', [ ip ]); - debug('verifyDomainConfig: Test A record added'); + log('verifyDomainConfig: Test A record added'); await del(domainObject, location, 'A', [ ip ]); - debug('verifyDomainConfig: Test A record removed again'); + log('verifyDomainConfig: Test A record removed again'); return credentials; } diff --git a/src/dns/hetznercloud.js b/src/dns/hetznercloud.js index aaf92428d..f752f9907 100644 --- a/src/dns/hetznercloud.js +++ b/src/dns/hetznercloud.js @@ -1,7 +1,7 @@ import assert from 'node:assert'; import BoxError from '../boxerror.js'; import constants from '../constants.js'; -import debugModule from 'debug'; +import logger from '../logger.js'; import dig from '../dig.js'; import dns from '../dns.js'; import promiseRetry from '../promise-retry.js'; @@ -9,7 +9,7 @@ import safe from 'safetydance'; import superagent from '@cloudron/superagent'; import waitForDns from './waitfordns.js'; -const debug = debugModule('box:dns/hetznercloud'); +const { log, trace } = logger('dns/hetznercloud'); // https://docs.hetzner.cloud/reference/cloud @@ -68,7 +68,7 @@ async function getRecords(domainConfig, zone, name, type) { assert.strictEqual(typeof name, 'string'); assert.strictEqual(typeof type, 'string'); - debug(`getRecords: getting dns records of ${zone.name} with ${name} and type ${type}`); + log(`getRecords: getting dns records of ${zone.name} with ${name} and type ${type}`); const [error, response] = await safe(superagent.get(`${ENDPOINT}/zones/${zone.id}/rrsets/${name}/${type.toUpperCase()}`) .set('Authorization', `Bearer ${domainConfig.token}`) @@ -88,7 +88,7 @@ async function waitForAction(domainConfig, id) { assert.strictEqual(typeof domainConfig, 'object'); assert.strictEqual(typeof id, 'number'); - await promiseRetry({ times: 100, interval: 1000, debug }, async () => { + await promiseRetry({ times: 100, interval: 1000, debug: log }, async () => { const [error, response] = await safe(superagent.get(`${ENDPOINT}/actions/${id}`) .set('Authorization', `Bearer ${domainConfig.token}`) .timeout(30 * 1000) @@ -141,7 +141,7 @@ async function upsert(domainObject, location, type, values) { zoneName = domainObject.zoneName, name = dns.getName(domainObject, location, type) || '@'; - debug(`upsert: ${name} for zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`); + log(`upsert: ${name} for zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`); const zone = await getZone(domainConfig, zoneName); const records = await getRecords(domainConfig, zone, name, type); @@ -213,17 +213,17 @@ async function verifyDomainConfig(domainObject) { // https://docs.hetzner.com/dns-console/dns/general/dns-overview#the-hetzner-online-name-servers-are if (!nameservers.every(function (n) { return n.toLowerCase().search(/hetzner|your-server|second-ns/) !== -1; })) { - debug('verifyDomainConfig: %j does not contain Hetzner NS', nameservers); + log('verifyDomainConfig: %j does not contain Hetzner NS', nameservers); if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Hetzner'); } const location = 'cloudrontestdns'; await upsert(domainObject, location, 'A', [ ip ]); - debug('verifyDomainConfig: Test A record added'); + log('verifyDomainConfig: Test A record added'); await del(domainObject, location, 'A', [ ip ]); - debug('verifyDomainConfig: Test A record removed again'); + log('verifyDomainConfig: Test A record removed again'); return credentials; } diff --git a/src/dns/inwx.js b/src/dns/inwx.js index 07ba8b2ba..43d23819a 100644 --- a/src/dns/inwx.js +++ b/src/dns/inwx.js @@ -2,13 +2,13 @@ import { ApiClient, Language } from 'domrobot-client'; import assert from 'node:assert'; import BoxError from '../boxerror.js'; import constants from '../constants.js'; -import debugModule from 'debug'; +import logger from '../logger.js'; import dig from '../dig.js'; import dns from '../dns.js'; import safe from 'safetydance'; import waitForDns from './waitfordns.js'; -const debug = debugModule('box:dns/inwx'); +const { log, trace } = logger('dns/inwx'); function formatError(response) { @@ -49,7 +49,7 @@ async function getDnsRecords(domainConfig, apiClient, zoneName, fqdn, type) { assert.strictEqual(typeof fqdn, 'string'); assert.strictEqual(typeof type, 'string'); - debug(`getDnsRecords: ${fqdn} in zone ${zoneName} of type ${type}`); + log(`getDnsRecords: ${fqdn} in zone ${zoneName} of type ${type}`); const [error, response] = await safe(apiClient.callApi('nameserver.info', { domain: zoneName, name: fqdn, type })); if (error) throw new BoxError(BoxError.NETWORK_ERROR, error); @@ -67,7 +67,7 @@ async function upsert(domainObject, location, type, values) { const domainConfig = domainObject.config, zoneName = domainObject.zoneName; - debug(`upsert: ${location} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`); + log(`upsert: ${location} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`); const apiClient = await login(domainConfig); const fqdn = dns.fqdn(location, domainObject.domain); @@ -140,7 +140,7 @@ async function del(domainObject, location, type, values) { const domainConfig = domainObject.config, zoneName = domainObject.zoneName; - debug(`del: ${location} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`); + log(`del: ${location} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`); const apiClient = await login(domainConfig); const fqdn = dns.fqdn(location, domainObject.domain); @@ -148,7 +148,7 @@ async function del(domainObject, location, type, values) { if (result.length === 0) return; const tmp = result.filter(function (record) { return values.some(function (value) { return value === record.content; }); }); - debug('del: %j', tmp); + log('del: %j', tmp); for (const r of tmp) { const [error, response] = await safe(apiClient.callApi('nameserver.deleteRecord', { id: r.id })); @@ -194,17 +194,17 @@ async function verifyDomainConfig(domainObject) { if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers'); if (!nameservers.every(function (n) { return n.toLowerCase().search(/inwx|xnameserver|domrobot/) !== -1; })) { - debug('verifyDomainConfig: %j does not contain INWX NS', nameservers); + log('verifyDomainConfig: %j does not contain INWX NS', nameservers); if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to INWX'); } const location = 'cloudrontestdns'; await upsert(domainObject, location, 'A', [ ip ]); - debug('verifyDomainConfig: Test A record added'); + log('verifyDomainConfig: Test A record added'); await del(domainObject, location, 'A', [ ip ]); - debug('verifyDomainConfig: Test A record removed again'); + log('verifyDomainConfig: Test A record removed again'); return credentials; } diff --git a/src/dns/linode.js b/src/dns/linode.js index 93d0c595f..db95fe2bb 100644 --- a/src/dns/linode.js +++ b/src/dns/linode.js @@ -1,14 +1,14 @@ import assert from 'node:assert'; import constants from '../constants.js'; import BoxError from '../boxerror.js'; -import debugModule from 'debug'; +import logger from '../logger.js'; import dig from '../dig.js'; import dns from '../dns.js'; import safe from 'safetydance'; import superagent from '@cloudron/superagent'; import waitForDns from './waitfordns.js'; -const debug = debugModule('box:dns/linode'); +const { log, trace } = logger('dns/linode'); const LINODE_ENDPOINT = 'https://api.linode.com/v4'; @@ -47,7 +47,7 @@ async function getZoneId(domainConfig, zoneName) { if (!zone || !zone.id) throw new BoxError(BoxError.NOT_FOUND, 'Zone not found'); - debug(`getZoneId: zone id of ${zoneName} is ${zone.id}`); + log(`getZoneId: zone id of ${zoneName} is ${zone.id}`); return zone.id; } @@ -58,7 +58,7 @@ async function getZoneRecords(domainConfig, zoneName, name, type) { assert.strictEqual(typeof name, 'string'); assert.strictEqual(typeof type, 'string'); - debug(`getInternal: getting dns records of ${zoneName} with ${name} and type ${type}`); + log(`getInternal: getting dns records of ${zoneName} with ${name} and type ${type}`); const zoneId = await getZoneId(domainConfig, zoneName); @@ -113,7 +113,7 @@ async function upsert(domainObject, location, type, values) { zoneName = domainObject.zoneName, name = dns.getName(domainObject, location, type) || ''; - debug('upsert: %s for zone %s of type %s with values %j', name, zoneName, type, values); + log('upsert: %s for zone %s of type %s with values %j', name, zoneName, type, values); const { zoneId, records } = await getZoneRecords(domainConfig, zoneName, name, type); let i = 0; // used to track available records to update instead of create @@ -176,7 +176,7 @@ async function upsert(domainObject, location, type, values) { .retry(5) .ok(() => true)); - if (error) debug(`upsert: error removing record ${records[j].id}: ${error.message}`); + if (error) log(`upsert: error removing record ${records[j].id}: ${error.message}`); } } @@ -244,17 +244,17 @@ async function verifyDomainConfig(domainObject) { if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers'); if (nameservers.map(function (n) { return n.toLowerCase(); }).indexOf('ns1.linode.com') === -1) { - debug('verifyDomainConfig: %j does not contains linode NS', nameservers); + log('verifyDomainConfig: %j does not contains linode NS', nameservers); if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Linode'); } const location = 'cloudrontestdns'; await upsert(domainObject, location, 'A', [ ip ]); - debug('verifyDomainConfig: Test A record added'); + log('verifyDomainConfig: Test A record added'); await del(domainObject, location, 'A', [ ip ]); - debug('verifyDomainConfig: Test A record removed again'); + log('verifyDomainConfig: Test A record removed again'); return credentials; } diff --git a/src/dns/manual.js b/src/dns/manual.js index b94e58af9..08c3cec04 100644 --- a/src/dns/manual.js +++ b/src/dns/manual.js @@ -1,12 +1,12 @@ import assert from 'node:assert'; import BoxError from '../boxerror.js'; -import debugModule from 'debug'; +import logger from '../logger.js'; import dig from '../dig.js'; import dns from '../dns.js'; import safe from 'safetydance'; import waitForDns from './waitfordns.js'; -const debug = debugModule('box:dns/manual'); +const { log, trace } = logger('dns/manual'); function removePrivateFields(domainObject) { @@ -24,7 +24,7 @@ async function upsert(domainObject, location, type, values) { assert.strictEqual(typeof type, 'string'); assert(Array.isArray(values)); - debug('upsert: %s for zone %s of type %s with values %j', location, domainObject.zoneName, type, values); + log('upsert: %s for zone %s of type %s with values %j', location, domainObject.zoneName, type, values); return; } diff --git a/src/dns/namecheap.js b/src/dns/namecheap.js index 12a0b8900..9799bac93 100644 --- a/src/dns/namecheap.js +++ b/src/dns/namecheap.js @@ -1,7 +1,7 @@ import assert from 'node:assert'; import BoxError from '../boxerror.js'; import constants from '../constants.js'; -import debugModule from 'debug'; +import logger from '../logger.js'; import dig from '../dig.js'; import dns from '../dns.js'; import network from '../network.js'; @@ -12,7 +12,7 @@ import util from 'node:util'; import waitForDns from './waitfordns.js'; import xml2js from 'xml2js'; -const debug = debugModule('box:dns/namecheap'); +const { log, trace } = logger('dns/namecheap'); const ENDPOINT = 'https://api.namecheap.com/xml.response'; @@ -130,7 +130,7 @@ async function upsert(domainObject, subdomain, type, values) { subdomain = dns.getName(domainObject, subdomain, type) || '@'; - debug('upsert: %s for zone %s of type %s with values %j', subdomain, zoneName, type, values); + log('upsert: %s for zone %s of type %s with values %j', subdomain, zoneName, type, values); const result = await getZone(domainConfig, zoneName); @@ -210,7 +210,7 @@ async function del(domainObject, subdomain, type, values) { subdomain = dns.getName(domainObject, subdomain, type) || '@'; - debug('del: %s for zone %s of type %s with values %j', subdomain, zoneName, type, values); + log('del: %s for zone %s of type %s with values %j', subdomain, zoneName, type, values); let result = await getZone(domainConfig, zoneName); if (result.length === 0) return; @@ -261,17 +261,17 @@ async function verifyDomainConfig(domainObject) { if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers'); if (nameservers.some(function (n) { return n.toLowerCase().indexOf('.registrar-servers.com') === -1; })) { - debug('verifyDomainConfig: %j does not contains NC NS', nameservers); + log('verifyDomainConfig: %j does not contains NC NS', nameservers); if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to NameCheap'); } const testSubdomain = 'cloudrontestdns'; await upsert(domainObject, testSubdomain, 'A', [ip]); - debug('verifyDomainConfig: Test A record added'); + log('verifyDomainConfig: Test A record added'); await del(domainObject, testSubdomain, 'A', [ip]); - debug('verifyDomainConfig: Test A record removed again'); + log('verifyDomainConfig: Test A record removed again'); return credentials; } diff --git a/src/dns/namecom.js b/src/dns/namecom.js index 86afe7397..37fe2f8cc 100644 --- a/src/dns/namecom.js +++ b/src/dns/namecom.js @@ -1,14 +1,14 @@ import assert from 'node:assert'; import BoxError from '../boxerror.js'; import constants from '../constants.js'; -import debugModule from 'debug'; +import logger from '../logger.js'; import dig from '../dig.js'; import dns from '../dns.js'; import safe from 'safetydance'; import superagent from '@cloudron/superagent'; import waitForDns from './waitfordns.js'; -const debug = debugModule('box:dns/namecom'); +const { log, trace } = logger('dns/namecom'); const NAMECOM_API = 'https://api.name.com/v4'; @@ -33,7 +33,7 @@ async function addRecord(domainConfig, zoneName, name, type, values) { assert.strictEqual(typeof type, 'string'); assert(Array.isArray(values)); - debug(`add: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`); + log(`add: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`); const data = { host: name, @@ -71,7 +71,7 @@ async function updateRecord(domainConfig, zoneName, recordId, name, type, values assert.strictEqual(typeof type, 'string'); assert(Array.isArray(values)); - debug(`update:${recordId} on ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`); + log(`update:${recordId} on ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`); const data = { host: name, @@ -107,7 +107,7 @@ async function getInternal(domainConfig, zoneName, name, type) { assert.strictEqual(typeof name, 'string'); assert.strictEqual(typeof type, 'string'); - debug(`getInternal: ${name} in zone ${zoneName} of type ${type}`); + log(`getInternal: ${name} in zone ${zoneName} of type ${type}`); const [error, response] = await safe(superagent.get(`${NAMECOM_API}/domains/${zoneName}/records`) .auth(domainConfig.username, domainConfig.token) @@ -144,7 +144,7 @@ async function upsert(domainObject, location, type, values) { zoneName = domainObject.zoneName, name = dns.getName(domainObject, location, type) || ''; - debug(`upsert: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`); + log(`upsert: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`); const result = await getInternal(domainConfig, zoneName, name, type); if (result.length === 0) return await addRecord(domainConfig, zoneName, name, type, values); @@ -176,7 +176,7 @@ async function del(domainObject, location, type, values) { zoneName = domainObject.zoneName, name = dns.getName(domainObject, location, type) || ''; - debug(`del: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`); + log(`del: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`); const result = await getInternal(domainConfig, zoneName, name, type); if (result.length === 0) return; @@ -227,17 +227,17 @@ async function verifyDomainConfig(domainObject) { if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers'); if (!nameservers.every(function (n) { return n.toLowerCase().indexOf('.name.com') !== -1; })) { - debug('verifyDomainConfig: %j does not contain Name.com NS', nameservers); + log('verifyDomainConfig: %j does not contain Name.com NS', nameservers); if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to name.com'); } const location = 'cloudrontestdns'; await upsert(domainObject, location, 'A', [ ip ]); - debug('verifyDomainConfig: Test A record added'); + log('verifyDomainConfig: Test A record added'); await del(domainObject, location, 'A', [ ip ]); - debug('verifyDomainConfig: Test A record removed again'); + log('verifyDomainConfig: Test A record removed again'); return credentials; } diff --git a/src/dns/netcup.js b/src/dns/netcup.js index 79814a982..dd80c49c4 100644 --- a/src/dns/netcup.js +++ b/src/dns/netcup.js @@ -1,14 +1,14 @@ import assert from 'node:assert'; import BoxError from '../boxerror.js'; import constants from '../constants.js'; -import debugModule from 'debug'; +import logger from '../logger.js'; import dig from '../dig.js'; import dns from '../dns.js'; import safe from 'safetydance'; import superagent from '@cloudron/superagent'; import waitForDns from './waitfordns.js'; -const debug = debugModule('box:dns/netcup'); +const { log, trace } = logger('dns/netcup'); const API_ENDPOINT = 'https://ccp.netcup.net/run/webservice/servers/endpoint.php?JSON'; @@ -53,7 +53,7 @@ async function getAllRecords(domainConfig, apiSessionId, zoneName) { assert.strictEqual(typeof apiSessionId, 'string'); assert.strictEqual(typeof zoneName, 'string'); - debug(`getAllRecords: getting dns records of ${zoneName}`); + log(`getAllRecords: getting dns records of ${zoneName}`); const data = { action: 'infoDnsRecords', @@ -82,7 +82,7 @@ async function upsert(domainObject, location, type, values) { zoneName = domainObject.zoneName, name = dns.getName(domainObject, location, type) || '@'; - debug('upsert: %s for zone %s of type %s with values %j', name, zoneName, type, values); + log('upsert: %s for zone %s of type %s with values %j', name, zoneName, type, values); const apiSessionId = await login(domainConfig); @@ -138,7 +138,7 @@ async function get(domainObject, location, type) { zoneName = domainObject.zoneName, name = dns.getName(domainObject, location, type) || '@'; - debug('get: %s for zone %s of type %s', name, zoneName, type); + log('get: %s for zone %s of type %s', name, zoneName, type); const apiSessionId = await login(domainConfig); @@ -158,7 +158,7 @@ async function del(domainObject, location, type, values) { zoneName = domainObject.zoneName, name = dns.getName(domainObject, location, type) || '@'; - debug('del: %s for zone %s of type %s with values %j', name, zoneName, type, values); + log('del: %s for zone %s of type %s with values %j', name, zoneName, type, values); const apiSessionId = await login(domainConfig); @@ -239,17 +239,17 @@ async function verifyDomainConfig(domainObject) { if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers'); if (!nameservers.every(function (n) { return n.toLowerCase().indexOf('dns.netcup.net') !== -1; })) { - debug('verifyDomainConfig: %j does not contains Netcup NS', nameservers); + log('verifyDomainConfig: %j does not contains Netcup NS', nameservers); if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Netcup'); } const location = 'cloudrontestdns'; await upsert(domainObject, location, 'A', [ ip ]); - debug('verifyDomainConfig: Test A record added'); + log('verifyDomainConfig: Test A record added'); await del(domainObject, location, 'A', [ ip ]); - debug('verifyDomainConfig: Test A record removed again'); + log('verifyDomainConfig: Test A record removed again'); return credentials; } diff --git a/src/dns/noop.js b/src/dns/noop.js index eb8234889..5eeeaee8e 100644 --- a/src/dns/noop.js +++ b/src/dns/noop.js @@ -1,7 +1,7 @@ import assert from 'node:assert'; -import debugModule from 'debug'; +import logger from '../logger.js'; -const debug = debugModule('box:dns/noop'); +const { log, trace } = logger('dns/noop'); function removePrivateFields(domainObject) { @@ -18,7 +18,7 @@ async function upsert(domainObject, location, type, values) { assert.strictEqual(typeof type, 'string'); assert(Array.isArray(values)); - debug('upsert: %s for zone %s of type %s with values %j', location, domainObject.zoneName, type, values); + log('upsert: %s for zone %s of type %s with values %j', location, domainObject.zoneName, type, values); return; } diff --git a/src/dns/ovh.js b/src/dns/ovh.js index 99ce8105e..5b059192e 100644 --- a/src/dns/ovh.js +++ b/src/dns/ovh.js @@ -1,14 +1,14 @@ import assert from 'node:assert'; import BoxError from '../boxerror.js'; import constants from '../constants.js'; -import debugModule from 'debug'; +import logger from '../logger.js'; import dig from '../dig.js'; import dns from '../dns.js'; import ovhClient from 'ovh'; import safe from 'safetydance'; import waitForDns from './waitfordns.js'; -const debug = debugModule('box:dns/ovh'); +const { log, trace } = logger('dns/ovh'); function formatError(error) { @@ -39,7 +39,7 @@ async function getDnsRecordIds(domainConfig, zoneName, name, type) { assert.strictEqual(typeof name, 'string'); assert.strictEqual(typeof type, 'string'); - debug(`get: ${name} in zone ${zoneName} of type ${type}`); + log(`get: ${name} in zone ${zoneName} of type ${type}`); const client = createClient(domainConfig); const [error, data] = await safe(client.requestPromised('GET', `/domain/zone/${zoneName}/record`, { fieldType: type, subDomain: name })); @@ -54,7 +54,7 @@ async function refreshZone(domainConfig, zoneName) { assert.strictEqual(typeof domainConfig, 'object'); assert.strictEqual(typeof zoneName, 'string'); - debug(`refresh: zone ${zoneName}`); + log(`refresh: zone ${zoneName}`); const client = createClient(domainConfig); const [error] = await safe(client.requestPromised('POST', `/domain/zone/${zoneName}/refresh`)); @@ -74,7 +74,7 @@ async function upsert(domainObject, location, type, values) { zoneName = domainObject.zoneName, name = dns.getName(domainObject, location, type) || ''; - debug(`upsert: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`); + log(`upsert: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`); const recordIds = await getDnsRecordIds(domainConfig, zoneName, name, type); @@ -115,7 +115,7 @@ async function upsert(domainObject, location, type, values) { } await refreshZone(domainConfig, zoneName); - debug('upsert: completed'); + log('upsert: completed'); } async function get(domainObject, location, type) { @@ -152,7 +152,7 @@ async function del(domainObject, location, type, values) { zoneName = domainObject.zoneName, name = dns.getName(domainObject, location, type) || ''; - debug(`del: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`); + log(`del: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`); const recordIds = await getDnsRecordIds(domainConfig, zoneName, name, type); @@ -211,17 +211,17 @@ async function verifyDomainConfig(domainObject) { // ovh.net, ovh.ca or anycast.me if (!nameservers.every(function (n) { return n.toLowerCase().search(/ovh|kimsufi|anycast/) !== -1; })) { - debug('verifyDomainConfig: %j does not contain OVH NS', nameservers); + log('verifyDomainConfig: %j does not contain OVH NS', nameservers); if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to OVH'); } const location = 'cloudrontestdns'; await upsert(domainObject, location, 'A', [ ip ]); - debug('verifyDomainConfig: Test A record added'); + log('verifyDomainConfig: Test A record added'); await del(domainObject, location, 'A', [ ip ]); - debug('verifyDomainConfig: Test A record removed again'); + log('verifyDomainConfig: Test A record removed again'); return credentials; } diff --git a/src/dns/porkbun.js b/src/dns/porkbun.js index 8d1786ded..c38d3fc23 100644 --- a/src/dns/porkbun.js +++ b/src/dns/porkbun.js @@ -1,7 +1,7 @@ import assert from 'node:assert'; import BoxError from '../boxerror.js'; import constants from '../constants.js'; -import debugModule from 'debug'; +import logger from '../logger.js'; import dig from '../dig.js'; import dns from '../dns.js'; import safe from 'safetydance'; @@ -9,7 +9,7 @@ import superagent from '@cloudron/superagent'; import timers from 'timers/promises'; import waitForDns from './waitfordns.js'; -const debug = debugModule('box:dns/porkbun'); +const { log, trace } = logger('dns/porkbun'); // Rate limit note: Porkbun return 503 when it hits rate limits. It's as low as 1 req/second @@ -44,7 +44,7 @@ async function getDnsRecords(domainConfig, zoneName, name, type) { assert.strictEqual(typeof name, 'string'); assert.strictEqual(typeof type, 'string'); - debug(`get: ${name} zone:${zoneName} type:${type}`); + log(`get: ${name} zone:${zoneName} type:${type}`); const data = { secretapikey: domainConfig.secretapikey, @@ -90,7 +90,7 @@ async function upsert(domainObject, location, type, values) { zoneName = domainObject.zoneName, name = dns.getName(domainObject, location, type) || ''; - debug(`upsert: ${name} zone:${zoneName} type:${type} values:${JSON.stringify(values)}`); + log(`upsert: ${name} zone:${zoneName} type:${type} values:${JSON.stringify(values)}`); await delDnsRecords(domainConfig, zoneName, name, type); @@ -119,7 +119,7 @@ async function upsert(domainObject, location, type, values) { if (response.body.status !== 'SUCCESS') throw new BoxError(BoxError.EXTERNAL_ERROR, `Invalid status in response: ${JSON.stringify(response.body)}`); if (!response.body.id) throw new BoxError(BoxError.EXTERNAL_ERROR, `Invalid id in response: ${JSON.stringify(response.body)}`); - debug(`upsert: created record with id ${response.body.id}`); + log(`upsert: created record with id ${response.body.id}`); } } @@ -146,7 +146,7 @@ async function del(domainObject, location, type, values) { zoneName = domainObject.zoneName, name = dns.getName(domainObject, location, type) || ''; - debug(`del: ${name} zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`); + log(`del: ${name} zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`); const data = { secretapikey: domainConfig.secretapikey, @@ -203,17 +203,17 @@ async function verifyDomainConfig(domainObject) { if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers'); if (!nameservers.every(function (n) { return n.toLowerCase().indexOf('.ns.porkbun.com') !== -1; })) { - debug('verifyDomainConfig: %j does not contain Porkbun NS', nameservers); + log('verifyDomainConfig: %j does not contain Porkbun NS', nameservers); if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Porkbun'); } const location = 'cloudrontestdns'; await upsert(domainObject, location, 'A', [ ip ]); - debug('verifyDomainConfig: Test A record added'); + log('verifyDomainConfig: Test A record added'); await del(domainObject, location, 'A', [ ip ]); - debug('verifyDomainConfig: Test A record removed again'); + log('verifyDomainConfig: Test A record removed again'); return credentials; } diff --git a/src/dns/route53.js b/src/dns/route53.js index d3211ca9b..4ac0be8a5 100644 --- a/src/dns/route53.js +++ b/src/dns/route53.js @@ -2,7 +2,7 @@ import assert from 'node:assert'; import BoxError from '../boxerror.js'; import { ConfiguredRetryStrategy } from '@smithy/util-retry'; import constants from '../constants.js'; -import debugModule from 'debug'; +import logger from '../logger.js'; import dig from '../dig.js'; import dns from '../dns.js'; import { Route53 } from '@aws-sdk/client-route-53'; @@ -10,7 +10,7 @@ import safe from 'safetydance'; import waitForDns from './waitfordns.js'; import _ from '../underscore.js'; -const debug = debugModule('box:dns/route53'); +const { log, trace } = logger('dns/route53'); function removePrivateFields(domainObject) { @@ -93,7 +93,7 @@ async function upsert(domainObject, location, type, values) { zoneName = domainObject.zoneName, fqdn = dns.fqdn(location, domainObject.domain); - debug('add: %s for zone %s of type %s with values %j', fqdn, zoneName, type, values); + log('add: %s for zone %s of type %s with values %j', fqdn, zoneName, type, values); const zone = await getZoneByName(domainConfig, zoneName); @@ -243,7 +243,7 @@ async function verifyDomainConfig(domainObject) { const zone = await getHostedZone(credentials, zoneName); if (!_.isEqual(zone.DelegationSet.NameServers.sort(), nameservers.sort())) { - debug('verifyDomainConfig: %j and %j do not match', nameservers, zone.DelegationSet.NameServers); + log('verifyDomainConfig: %j and %j do not match', nameservers, zone.DelegationSet.NameServers); if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Route53'); } @@ -251,13 +251,13 @@ async function verifyDomainConfig(domainObject) { const newDomainObject = Object.assign({ }, domainObject, { config: credentials }); await upsert(newDomainObject, location, 'A', [ ip ]); - debug('verifyDomainConfig: Test A record added'); + log('verifyDomainConfig: Test A record added'); await get(newDomainObject, location, 'A'); - debug('verifyDomainConfig: Can list record sets'); + log('verifyDomainConfig: Can list record sets'); await del(newDomainObject, location, 'A', [ ip ]); - debug('verifyDomainConfig: Test A record removed again'); + log('verifyDomainConfig: Test A record removed again'); return credentials; } diff --git a/src/dns/vultr.js b/src/dns/vultr.js index 60b47f667..47cd42faf 100644 --- a/src/dns/vultr.js +++ b/src/dns/vultr.js @@ -1,14 +1,14 @@ import assert from 'node:assert'; import constants from '../constants.js'; import BoxError from '../boxerror.js'; -import debugModule from 'debug'; +import logger from '../logger.js'; import dig from '../dig.js'; import dns from '../dns.js'; import safe from 'safetydance'; import superagent from '@cloudron/superagent'; import waitForDns from './waitfordns.js'; -const debug = debugModule('box:dns/vultr'); +const { log, trace } = logger('dns/vultr'); const VULTR_ENDPOINT = 'https://api.vultr.com/v2'; @@ -32,7 +32,7 @@ async function getZoneRecords(domainConfig, zoneName, name, type) { assert.strictEqual(typeof name, 'string'); assert.strictEqual(typeof type, 'string'); - debug(`getInternal: getting dns records of ${zoneName} with ${name} and type ${type}`); + log(`getInternal: getting dns records of ${zoneName} with ${name} and type ${type}`); const per_page = 100; let cursor = null, records = []; @@ -80,7 +80,7 @@ async function upsert(domainObject, location, type, values) { zoneName = domainObject.zoneName, name = dns.getName(domainObject, location, type) || ''; - debug('upsert: %s for zone %s of type %s with values %j', name, zoneName, type, values); + log('upsert: %s for zone %s of type %s with values %j', name, zoneName, type, values); const records = await getZoneRecords(domainConfig, zoneName, name, type); @@ -143,10 +143,10 @@ async function upsert(domainObject, location, type, values) { .timeout(30 * 1000) .retry(5)); - if (error) debug(`upsert: error removing record ${records[j].id}: ${error.message}`); + if (error) log(`upsert: error removing record ${records[j].id}: ${error.message}`); } - debug('upsert: completed with recordIds:%j', recordIds); + log('upsert: completed with recordIds:%j', recordIds); } async function del(domainObject, location, type, values) { @@ -214,17 +214,17 @@ async function verifyDomainConfig(domainObject) { if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers'); if (nameservers.map(function (n) { return n.toLowerCase(); }).indexOf('ns1.vultr.com') === -1) { - debug('verifyDomainConfig: %j does not contains vultr NS', nameservers); + log('verifyDomainConfig: %j does not contains vultr NS', nameservers); if (!domainConfig.customNameservers) throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Vultr'); } const location = 'cloudrontestdns'; await upsert(domainObject, location, 'A', [ ip ]); - debug('verifyDomainConfig: Test A record added'); + log('verifyDomainConfig: Test A record added'); await del(domainObject, location, 'A', [ ip ]); - debug('verifyDomainConfig: Test A record removed again'); + log('verifyDomainConfig: Test A record removed again'); return credentials; } diff --git a/src/dns/waitfordns.js b/src/dns/waitfordns.js index 1bc684a52..5754f6fe8 100644 --- a/src/dns/waitfordns.js +++ b/src/dns/waitfordns.js @@ -1,13 +1,13 @@ import assert from 'node:assert'; import BoxError from '../boxerror.js'; -import debugModule from 'debug'; +import logger from '../logger.js'; import dig from '../dig.js'; import dns from 'node:dns'; import promiseRetry from '../promise-retry.js'; import safe from 'safetydance'; import _ from '../underscore.js'; -const debug = debugModule('box:dns/waitfordns'); +const { log, trace } = logger('dns/waitfordns'); async function resolveIp(hostname, type, options) { assert.strictEqual(typeof hostname, 'string'); @@ -15,17 +15,17 @@ async function resolveIp(hostname, type, options) { assert.strictEqual(typeof options, 'object'); // try A record at authoritative server - debug(`resolveIp: Checking ${type} for ${hostname} at ${options.server}`); + log(`resolveIp: Checking ${type} for ${hostname} at ${options.server}`); const [error, results] = await safe(dig.resolve(hostname, type, options)); if (!error && results.length !== 0) return results; // try CNAME record at authoritative server - debug(`resolveIp: No ${type}. Checking CNAME for ${hostname} at ${options.server}`); + log(`resolveIp: No ${type}. Checking CNAME for ${hostname} at ${options.server}`); const cnameResults = await dig.resolve(hostname, 'CNAME', options); if (cnameResults.length === 0) return cnameResults; // recurse lookup the CNAME record - debug(`resolveIp: found CNAME for ${hostname}. resolving ${cnameResults[0]}`); + log(`resolveIp: found CNAME for ${hostname}. resolving ${cnameResults[0]}`); return await dig.resolve(cnameResults[0], type, _.omit(options, ['server'])); } @@ -40,7 +40,7 @@ async function isChangeSynced(hostname, type, value, nameserver) { const [error6, nsIPv6s] = await safe(dig.resolve(nameserver, 'AAAA', { timeout: 5000 })); if (error4 && error6) { - debug(`isChangeSynced: cannot resolve NS ${nameserver}`); // NS doesn't resolve at all; it's fine + log(`isChangeSynced: cannot resolve NS ${nameserver}`); // NS doesn't resolve at all; it's fine return true; } @@ -54,13 +54,13 @@ async function isChangeSynced(hostname, type, value, nameserver) { const [error, answer] = await safe(resolver); // CONNREFUSED - when there is no ipv4/ipv6 connectivity. REFUSED - server won't answer maybe by policy if (error && (error.code === dns.TIMEOUT || error.code === dns.REFUSED || error.code === dns.CONNREFUSED)) { - debug(`isChangeSynced: NS ${nameserver} (${nsIp}) not resolving ${hostname} (${type}): ${error}. Ignoring`); + log(`isChangeSynced: NS ${nameserver} (${nsIp}) not resolving ${hostname} (${type}): ${error}. Ignoring`); status[i] = true; // should be ok if dns server is down continue; } if (error) { - debug(`isChangeSynced: NS ${nameserver} (${nsIp}) errored when resolve ${hostname} (${type}): ${error}`); + log(`isChangeSynced: NS ${nameserver} (${nsIp}) errored when resolve ${hostname} (${type}): ${error}`); status[i] = false; continue; } @@ -72,7 +72,7 @@ async function isChangeSynced(hostname, type, value, nameserver) { match = answer.some(function (a) { return value === a.join(''); }); } - debug(`isChangeSynced: ${hostname} (${type}) was resolved to ${answer} at NS ${nameserver} (${nsIp}). Expecting ${value}. Match ${match}`); + log(`isChangeSynced: ${hostname} (${type}) was resolved to ${answer} at NS ${nameserver} (${nsIp}). Expecting ${value}. Match ${match}`); status[i] = match; } @@ -87,21 +87,21 @@ async function waitForDns(hostname, zoneName, type, value, options) { assert.strictEqual(typeof value, 'string'); assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 } - debug(`waitForDns: waiting for ${hostname} to be ${value} in zone ${zoneName}`); + log(`waitForDns: waiting for ${hostname} to be ${value} in zone ${zoneName}`); - await promiseRetry(Object.assign({ debug }, options), async function () { + await promiseRetry(Object.assign({ debug: log }, options), async function () { const nameservers = await dig.resolve(zoneName, 'NS', { timeout: 5000 }); if (!nameservers) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Unable to get nameservers'); - debug(`waitForDns: nameservers are ${JSON.stringify(nameservers)}`); + log(`waitForDns: nameservers are ${JSON.stringify(nameservers)}`); for (const nameserver of nameservers) { const synced = await isChangeSynced(hostname, type, value, nameserver); - debug(`waitForDns: ${hostname} at ns ${nameserver}: ${synced ? 'done' : 'not done'} `); + log(`waitForDns: ${hostname} at ns ${nameserver}: ${synced ? 'done' : 'not done'} `); if (!synced) throw new BoxError(BoxError.EXTERNAL_ERROR, 'ETRYAGAIN'); } }); - debug(`waitForDns: ${hostname} has propagated`); + log(`waitForDns: ${hostname} has propagated`); } export default waitForDns; diff --git a/src/dns/wildcard.js b/src/dns/wildcard.js index a6c7c5bdc..a6d2091b1 100644 --- a/src/dns/wildcard.js +++ b/src/dns/wildcard.js @@ -1,13 +1,13 @@ import assert from 'node:assert'; import BoxError from '../boxerror.js'; -import debugModule from 'debug'; +import logger from '../logger.js'; import dig from '../dig.js'; import dns from '../dns.js'; import network from '../network.js'; import safe from 'safetydance'; import waitForDns from './waitfordns.js'; -const debug = debugModule('box:dns/manual'); +const { log, trace } = logger('dns/manual'); function removePrivateFields(domainObject) { @@ -24,7 +24,7 @@ async function upsert(domainObject, location, type, values) { assert.strictEqual(typeof type, 'string'); assert(Array.isArray(values)); - debug('upsert: %s for zone %s of type %s with values %j', location, domainObject.zoneName, type, values); + log('upsert: %s for zone %s of type %s with values %j', location, domainObject.zoneName, type, values); return; } diff --git a/src/docker.js b/src/docker.js index 180e4e384..ecf0fbab4 100644 --- a/src/docker.js +++ b/src/docker.js @@ -3,7 +3,7 @@ import assert from 'node:assert'; import BoxError from './boxerror.js'; import constants from './constants.js'; import dashboard from './dashboard.js'; -import debugModule from 'debug'; +import logger from './logger.js'; import Docker from 'dockerode'; import dockerRegistries from './dockerregistries.js'; import fs from 'node:fs'; @@ -17,7 +17,7 @@ import safe from 'safetydance'; import timers from 'timers/promises'; import volumes from './volumes.js'; -const debug = debugModule('box:docker'); +const { log, trace } = logger('docker'); const shell = shellModule('docker'); @@ -94,7 +94,7 @@ async function pullImage(imageRef) { const authConfig = await getAuthConfig(imageRef); - debug(`pullImage: will pull ${imageRef}. auth: ${authConfig ? 'yes' : 'no'}`); + log(`pullImage: will pull ${imageRef}. auth: ${authConfig ? 'yes' : 'no'}`); const [error, stream] = await safe(gConnection.pull(imageRef, { authconfig: authConfig })); if (error && error.statusCode === 404) throw new BoxError(BoxError.NOT_FOUND, `Unable to pull image ${imageRef}. message: ${error.message} statusCode: ${error.statusCode}`); @@ -107,17 +107,17 @@ async function pullImage(imageRef) { let layerError = null; stream.on('data', function (chunk) { const data = safe.JSON.parse(chunk) || { }; - debug('pullImage: %j', data); + log('pullImage: %j', data); // The data.status here is useless because this is per layer as opposed to per image if (!data.status && data.error) { // data is { errorDetail: { message: xx } , error: xx } - debug(`pullImage error ${imageRef}: ${data.errorDetail.message}`); + log(`pullImage error ${imageRef}: ${data.errorDetail.message}`); layerError = data.errorDetail; } }); stream.on('end', function () { - debug(`downloaded image ${imageRef} . error: ${!!layerError}`); + log(`downloaded image ${imageRef} . error: ${!!layerError}`); if (!layerError) return resolve(); @@ -125,7 +125,7 @@ async function pullImage(imageRef) { }); stream.on('error', function (streamError) { // this is only hit for stream error and not for some download error - debug(`error pulling image ${imageRef}: %o`, streamError); + log(`error pulling image ${imageRef}: %o`, streamError); reject(new BoxError(BoxError.DOCKER_ERROR, streamError.message)); }); }); @@ -135,7 +135,7 @@ async function buildImage(dockerImage, sourceArchiveFilePath) { assert.strictEqual(typeof dockerImage, 'string'); assert.strictEqual(typeof sourceArchiveFilePath, 'string'); - debug(`buildImage: building ${dockerImage} from ${sourceArchiveFilePath}`); + log(`buildImage: building ${dockerImage} from ${sourceArchiveFilePath}`); const buildOptions = { t: dockerImage }; const [listError, listOut] = await safe(shell.spawn('tar', ['-tzf', sourceArchiveFilePath], { encoding: 'utf8' })); @@ -146,7 +146,7 @@ async function buildImage(dockerImage, sourceArchiveFilePath) { }); if (dockerfileCloudronPath) { buildOptions.dockerfile = dockerfileCloudronPath.replace(/\/$/, ''); - debug(`buildImage: using ${buildOptions.dockerfile}`); + log(`buildImage: using ${buildOptions.dockerfile}`); } } @@ -164,23 +164,23 @@ async function buildImage(dockerImage, sourceArchiveFilePath) { buildError = data.errorDetail || { message: data.error }; } else { const message = (data.stream || data.status || data.aux?.ID || '').replace(/\n$/, ''); - if (message) debug('buildImage: ' + message); + if (message) log('buildImage: ' + message); } }); stream.on('end', () => { if (buildError) { - debug(`buildImage: error ${buildError}`); + log(`buildImage: error ${buildError}`); return reject(new BoxError(buildError.message.includes('no space') ? BoxError.FS_ERROR : BoxError.DOCKER_ERROR, buildError.message)); } else { - debug(`buildImage: success ${dockerImage}`); + log(`buildImage: success ${dockerImage}`); } resolve(); }); stream.on('error', (streamError) => { - debug(`buildImage: error building image ${dockerImage}: %o`, streamError); + log(`buildImage: error building image ${dockerImage}: %o`, streamError); reject(new BoxError(BoxError.DOCKER_ERROR, streamError.message)); }); }); @@ -329,7 +329,7 @@ async function restartContainer(containerId) { async function stopContainer(containerId) { assert.strictEqual(typeof containerId, 'string'); - debug(`stopContainer: stopping container ${containerId}`); + log(`stopContainer: stopping container ${containerId}`); const container = gConnection.getContainer(containerId); @@ -347,7 +347,7 @@ async function stopContainer(containerId) { async function deleteContainer(containerId) { // id can also be name assert.strictEqual(typeof containerId, 'string'); - debug(`deleteContainer: deleting ${containerId}`); + log(`deleteContainer: deleting ${containerId}`); const container = gConnection.getContainer(containerId); @@ -360,7 +360,7 @@ async function deleteContainer(containerId) { // id can also be name if (error && error.statusCode === 404) return; if (error) { - debug('Error removing container %s : %o', containerId, error); + log('Error removing container %s : %o', containerId, error); throw new BoxError(BoxError.DOCKER_ERROR, error); } } @@ -405,14 +405,14 @@ async function deleteImage(imageRef) { // registry v1 used to pull down all *tags*. this meant that deleting image by tag was not enough (since that // just removes the tag). we used to remove the image by id. this is not required anymore because aliases are // not created anymore after https://github.com/docker/docker/pull/10571 - debug(`deleteImage: removing ${imageRef}`); + log(`deleteImage: removing ${imageRef}`); const [error] = await safe(gConnection.getImage(imageRef.replace(/@sha256:.*/,'')).remove(removeOptions)); // can't have the manifest id. won't remove anythin if (error && error.statusCode === 400) return; // invalid image format. this can happen if user installed with a bad --docker-image if (error && error.statusCode === 404) return; // not found if (error && error.statusCode === 409) return; // another container using the image if (error) { - debug(`Error removing image ${imageRef} : %o`, error); + log(`Error removing image ${imageRef} : %o`, error); throw new BoxError(BoxError.DOCKER_ERROR, error); } } @@ -432,7 +432,7 @@ async function inspect(containerId) { async function downloadImage(manifest) { assert.strictEqual(typeof manifest, 'object'); - debug(`downloadImage: ${manifest.dockerImage}`); + log(`downloadImage: ${manifest.dockerImage}`); const image = gConnection.getImage(manifest.dockerImage); @@ -441,7 +441,7 @@ async function downloadImage(manifest) { const parsedManifestRef = parseImageRef(manifest.dockerImage); - await promiseRetry({ times: 10, interval: 5000, debug, retry: (pullError) => pullError.reason !== BoxError.FS_ERROR }, async () => { + await promiseRetry({ times: 10, interval: 5000, debug: log, retry: (pullError) => pullError.reason !== BoxError.FS_ERROR }, async () => { // custom (non appstore) image if (parsedManifestRef.registry !== null || !parsedManifestRef.fullRepositoryName.startsWith('cloudron/')) return await pullImage(manifest.dockerImage); @@ -457,9 +457,9 @@ async function downloadImage(manifest) { if (pullError || !upstreamRef) throw new BoxError(BoxError.DOCKER_ERROR, `Unable to pull ${manifest.dockerImage} from dockerhub or quay: ${pullError?.message}`); // retag the downloaded image to not have the registry name. this prevents 'docker run' from redownloading it - debug(`downloadImage: tagging ${upstreamRef} as ${parsedManifestRef.fullRepositoryName}:${parsedManifestRef.tag}`); + log(`downloadImage: tagging ${upstreamRef} as ${parsedManifestRef.fullRepositoryName}:${parsedManifestRef.tag}`); await gConnection.getImage(upstreamRef).tag({ repo: parsedManifestRef.fullRepositoryName, tag: parsedManifestRef.tag }); - debug(`downloadImage: untagging ${upstreamRef}`); + log(`downloadImage: untagging ${upstreamRef}`); await deleteImage(upstreamRef); }); } @@ -731,7 +731,7 @@ async function createSubcontainer(app, name, cmd, options) { containerOptions.HostConfig.Devices = Object.keys(app.devices).map((d) => { if (!safe.fs.existsSync(d)) { - debug(`createSubcontainer: device ${d} does not exist. Skipping...`); + log(`createSubcontainer: device ${d} does not exist. Skipping...`); return null; } diff --git a/src/dockerproxy.js b/src/dockerproxy.js index 0b72abde8..a8c3eff8d 100644 --- a/src/dockerproxy.js +++ b/src/dockerproxy.js @@ -2,7 +2,7 @@ import apps from './apps.js'; import assert from 'node:assert'; import constants from './constants.js'; import express from 'express'; -import debugModule from 'debug'; +import logger from './logger.js'; import http from 'node:http'; import { HttpError } from '@cloudron/connect-lastmile'; import middleware from './middleware/index.js'; @@ -13,7 +13,7 @@ import safe from 'safetydance'; import util from 'node:util'; import volumes from './volumes.js'; -const debug = debugModule('box:dockerproxy'); +const { log, trace } = logger('dockerproxy'); let gHttpServer = null; @@ -49,7 +49,7 @@ function attachDockerRequest(req, res, next) { // Force node to send out the headers, this is required for the /container/wait api to make the docker cli proceed res.write(' '); - dockerResponse.on('error', function (error) { debug('dockerResponse error: %o', error); }); + dockerResponse.on('error', function (error) { log('dockerResponse error: %o', error); }); dockerResponse.pipe(res, { end: true }); }); @@ -66,7 +66,7 @@ async function containersCreate(req, res, next) { const appDataDir = path.join(paths.APPS_DATA_DIR, req.resources.app.id, 'data'); - debug('containersCreate: original bind mounts:', req.body.HostConfig.Binds); + log('containersCreate: original bind mounts:', req.body.HostConfig.Binds); const [error, result] = await safe(volumes.list()); if (error) return next(new HttpError(500, `Error listing volumes: ${error.message}`)); @@ -82,14 +82,14 @@ async function containersCreate(req, res, next) { const volumeName = bind.match(new RegExp('/media/([^:/]+)/?'))[1]; const volume = volumesByName[volumeName]; if (volume) binds.push(bind.replace(new RegExp(`^/media/${volumeName}`), volume.hostPath)); - else debug(`containersCreate: dropped unknown volume ${volumeName}`); + else log(`containersCreate: dropped unknown volume ${volumeName}`); } else { req.dockerRequest.abort(); return next(new HttpError(400, 'Binds must be under /app/data/ or /media')); } } - debug('containersCreate: rewritten bind mounts:', binds); + log('containersCreate: rewritten bind mounts:', binds); safe.set(req.body, 'HostConfig.Binds', binds); const plainBody = JSON.stringify(req.body); @@ -125,7 +125,7 @@ async function start() { if (constants.TEST) { proxyServer.use(function (req, res, next) { - debug('proxying: ' + req.method, req.url); + log('proxying: ' + req.method, req.url); next(); }); } @@ -177,7 +177,7 @@ async function start() { }); }); - debug(`start: listening on 172.18.0.1:${constants.DOCKER_PROXY_PORT}`); + log(`start: listening on 172.18.0.1:${constants.DOCKER_PROXY_PORT}`); await util.promisify(gHttpServer.listen.bind(gHttpServer))(constants.DOCKER_PROXY_PORT, '172.18.0.1'); } diff --git a/src/domains.js b/src/domains.js index f0c91350c..7281ae470 100644 --- a/src/domains.js +++ b/src/domains.js @@ -4,7 +4,7 @@ import constants from './constants.js'; import crypto from 'node:crypto'; import dashboard from './dashboard.js'; import database from './database.js'; -import debugModule from 'debug'; +import logger from './logger.js'; import eventlog from './eventlog.js'; import mailServer from './mailserver.js'; import notifications from './notifications.js'; @@ -36,7 +36,7 @@ import dnsManual from './dns/manual.js'; import dnsPorkbun from './dns/porkbun.js'; import dnsWildcard from './dns/wildcard.js'; -const debug = debugModule('box:domains'); +const { log, trace } = logger('domains'); const DOMAINS_FIELDS = [ 'domain', 'zoneName', 'provider', 'configJson', 'tlsConfigJson', 'wellKnownJson', 'fallbackCertificateJson' ].join(','); @@ -187,7 +187,7 @@ async function add(domain, data, auditSource) { await eventlog.add(eventlog.ACTION_DOMAIN_ADD, auditSource, { domain, zoneName, provider }); - safe(mailServer.onDomainAdded(domain), { debug }); // background + safe(mailServer.onDomainAdded(domain), { debug: log }); // background } async function get(domain) { @@ -341,7 +341,7 @@ async function getDomainObjectMap() { async function checkConfigs(auditSource) { assert.strictEqual(typeof auditSource, 'object'); - debug(`checkConfig: validating domain configs`); + log(`checkConfig: validating domain configs`); for (const domainObject of await list()) { if (domainObject.provider === 'noop' || domainObject.provider === 'manual' || domainObject.provider === 'wildcard') { @@ -366,7 +366,7 @@ async function checkConfigs(auditSource) { errorMessage = `General error: ${error.message}`; } - debug(`checkConfig: ${domainObject.domain} is not configured properly`, error); + log(`checkConfig: ${domainObject.domain} is not configured properly`, error); await notifications.pin(notifications.TYPE_DOMAIN_CONFIG_CHECK_FAILED, `Domain ${domainObject.domain} is not configured properly`, errorMessage, { context: domainObject.domain }); diff --git a/src/dyndns.js b/src/dyndns.js index 06a211763..784b6e77c 100644 --- a/src/dyndns.js +++ b/src/dyndns.js @@ -1,7 +1,7 @@ import apps from './apps.js'; import assert from 'node:assert'; import dashboard from './dashboard.js'; -import debugModule from 'debug'; +import logger from './logger.js'; import dns from './dns.js'; import eventlog from './eventlog.js'; import fs from 'node:fs'; @@ -11,7 +11,7 @@ import paths from './paths.js'; import safe from 'safetydance'; import tasks from './tasks.js'; -const debug = debugModule('box:dyndns'); +const { log, trace } = logger('dyndns'); // FIXME: this races with apptask. can result in a conflict if apptask is doing some dns operation and this code changes entries @@ -26,11 +26,11 @@ async function refreshDns(auditSource) { const ipv6Changed = ipv6 && info.ipv6 !== ipv6; // both should be RFC 5952 format if (!ipv4Changed && !ipv6Changed) { - debug(`refreshDns: no change in IP. ipv4: ${ipv4} ipv6: ${ipv6}`); + log(`refreshDns: no change in IP. ipv4: ${ipv4} ipv6: ${ipv6}`); return; } - debug(`refreshDns: updating IP from ${info.ipv4} to ipv4: ${ipv4} (changed: ${ipv4Changed}) ipv6: ${ipv6} (changed: ${ipv6Changed})`); + log(`refreshDns: updating IP from ${info.ipv4} to ipv4: ${ipv4} (changed: ${ipv4Changed}) ipv6: ${ipv6} (changed: ${ipv6Changed})`); const taskId = await tasks.add(tasks.TASK_SYNC_DYNDNS, [ ipv4Changed ? ipv4 : null, ipv6Changed ? ipv6 : null, auditSource ]); // background @@ -57,15 +57,15 @@ async function sync(ipv4, ipv6, auditSource, progressCallback) { let percent = 5; const { domain:dashboardDomain, fqdn:dashboardFqdn, subdomain:dashboardSubdomain } = await dashboard.getLocation(); progressCallback({ percent, message: `Updating dashboard location ${dashboardFqdn}`}); - if (ipv4) await safe(dns.upsertDnsRecords(dashboardSubdomain, dashboardDomain, 'A', [ ipv4 ]), { debug }); - if (ipv6) await safe(dns.upsertDnsRecords(dashboardSubdomain, dashboardDomain, 'AAAA', [ ipv6 ]), { debug }); + if (ipv4) await safe(dns.upsertDnsRecords(dashboardSubdomain, dashboardDomain, 'A', [ ipv4 ]), { debug: log }); + if (ipv6) await safe(dns.upsertDnsRecords(dashboardSubdomain, dashboardDomain, 'AAAA', [ ipv6 ]), { debug: log }); const { domain:mailDomain, fqdn:mailFqdn, subdomain:mailSubdomain } = await mailServer.getLocation(); percent += 10; progressCallback({ percent, message: `Updating mail location ${mailFqdn}`}); if (dashboardFqdn !== mailFqdn) { - if (ipv4) await safe(dns.upsertDnsRecords(mailSubdomain, mailDomain, 'A', [ ipv4 ]), { debug }); - if (ipv6) await safe(dns.upsertDnsRecords(mailSubdomain, mailDomain, 'AAAA', [ ipv6 ]), { debug }); + if (ipv4) await safe(dns.upsertDnsRecords(mailSubdomain, mailDomain, 'A', [ ipv4 ]), { debug: log }); + if (ipv6) await safe(dns.upsertDnsRecords(mailSubdomain, mailDomain, 'AAAA', [ ipv6 ]), { debug: log }); } const result = await apps.list(); @@ -79,8 +79,8 @@ async function sync(ipv4, ipv6, auditSource, progressCallback) { .concat(app.aliasDomains); for (const location of locations) { - if (ipv4) await safe(dns.upsertDnsRecords(location.subdomain, location.domain, 'A', [ ipv4 ]), { debug }); - if (ipv6) await safe(dns.upsertDnsRecords(location.subdomain, location.domain, 'AAAA', [ ipv6 ], { debug })); + if (ipv4) await safe(dns.upsertDnsRecords(location.subdomain, location.domain, 'A', [ ipv4 ]), { debug: log }); + if (ipv6) await safe(dns.upsertDnsRecords(location.subdomain, location.domain, 'AAAA', [ ipv6 ], { debug: log })); } } diff --git a/src/eventlog.js b/src/eventlog.js index 3aa68bb33..14fe3fcb1 100644 --- a/src/eventlog.js +++ b/src/eventlog.js @@ -1,12 +1,12 @@ import assert from 'node:assert'; import crypto from 'node:crypto'; import database from './database.js'; -import debugModule from 'debug'; +import logger from './logger.js'; import mysql from 'mysql2'; import notifications from './notifications.js'; import safe from 'safetydance'; -const debug = debugModule('box:eventlog'); +const { log, trace } = logger('eventlog'); const ACTION_ACTIVATE = 'cloudron.activate'; const ACTION_USER_LOGIN = 'user.login'; @@ -119,7 +119,7 @@ async function cleanup(options) { assert.strictEqual(typeof options, 'object'); const creationTime = options.creationTime; - debug(`cleanup: pruning events. creationTime: ${creationTime.toString()}`); + log(`cleanup: pruning events. creationTime: ${creationTime.toString()}`); // only these actions are pruned const actions = [ diff --git a/src/externalldap.js b/src/externalldap.js index 5e7471a8c..0be2b698b 100644 --- a/src/externalldap.js +++ b/src/externalldap.js @@ -3,7 +3,7 @@ import AuditSource from './auditsource.js'; import BoxError from './boxerror.js'; import constants from './constants.js'; import cron from './cron.js'; -import debugModule from 'debug'; +import logger from './logger.js'; import eventlog from './eventlog.js'; import groups from './groups.js'; import ldap from 'ldapjs'; @@ -13,7 +13,7 @@ import tasks from './tasks.js'; import users from './users.js'; import util from 'node:util'; -const debug = debugModule('box:externalldap'); +const { log, trace } = logger('externalldap'); function removePrivateFields(ldapConfig) { @@ -38,7 +38,7 @@ function translateUser(ldapConfig, ldapUser) { }; if (!user.username || !user.email || !user.displayName) { - debug(`[Invalid LDAP user] username=${user.username} email=${user.email} displayName=${user.displayName}`); + log(`[Invalid LDAP user] username=${user.username} email=${user.email} displayName=${user.displayName}`); return null; } @@ -79,7 +79,7 @@ async function getClient(config, options) { return await new Promise((resolve, reject) => { // ensure we don't just crash client.on('error', function (error) { // don't reject, we must have gotten a bind error - debug('getClient: ExternalLdap client error:', error); + log('getClient: ExternalLdap client error:', error); }); // skip bind auth if none exist or if not wanted @@ -133,16 +133,16 @@ async function supportsPagination(client) { const result = await clientSearch(client, '', searchOptions); const controls = result.supportedControl; if (!controls || !Array.isArray(controls)) { - debug('supportsPagination: no supportedControl attribute returned'); + log('supportsPagination: no supportedControl attribute returned'); return false; } if (!controls.includes(ldap.PagedResultsControl.OID)) { - debug('supportsPagination: server does not support pagination. Available controls:', controls); + log('supportsPagination: server does not support pagination. Available controls:', controls); return false; } - debug('supportsPagination: server supports pagination'); + log('supportsPagination: server supports pagination'); return true; } @@ -150,7 +150,7 @@ async function ldapGetByDN(config, dn) { assert.strictEqual(typeof config, 'object'); assert.strictEqual(typeof dn, 'string'); - debug(`ldapGetByDN: Get object at ${dn}`); + log(`ldapGetByDN: Get object at ${dn}`); const client = await getClient(config, { bind: true }); const paged = await supportsPagination(client); @@ -343,7 +343,7 @@ async function startSyncer() { if (config.provider === 'noop') throw new BoxError(BoxError.BAD_STATE, 'not enabled'); const taskId = await tasks.add(tasks.TASK_SYNC_EXTERNAL_LDAP, []); - safe(tasks.startTask(taskId, {}), { debug }); // background + safe(tasks.startTask(taskId, {}), { debug: log }); // background return taskId; } @@ -353,7 +353,7 @@ async function syncUsers(config, progressCallback) { const ldapUsers = await ldapUserSearch(config, {}); - debug(`syncUsers: Found ${ldapUsers.length} users`); + log(`syncUsers: Found ${ldapUsers.length} users`); let percent = 10; const step = 28 / (ldapUsers.length + 1); @@ -369,27 +369,27 @@ async function syncUsers(config, progressCallback) { const user = await users.getByUsername(ldapUser.username); if (!user) { - debug(`syncUsers: [adding user] username=${ldapUser.username} email=${ldapUser.email} displayName=${ldapUser.displayName}`); + log(`syncUsers: [adding user] username=${ldapUser.username} email=${ldapUser.email} displayName=${ldapUser.displayName}`); const [userAddError] = await safe(users.add(ldapUser.email, { username: ldapUser.username, password: null, displayName: ldapUser.displayName, source: 'ldap' }, AuditSource.EXTERNAL_LDAP)); - if (userAddError) debug('syncUsers: Failed to create user. %j %o', ldapUser, userAddError); + if (userAddError) log('syncUsers: Failed to create user. %j %o', ldapUser, userAddError); } else if (user.source !== 'ldap') { - debug(`syncUsers: [mapping user] username=${ldapUser.username} email=${ldapUser.email} displayName=${ldapUser.displayName}`); + log(`syncUsers: [mapping user] username=${ldapUser.username} email=${ldapUser.email} displayName=${ldapUser.displayName}`); const [userMappingError] = await safe(users.update(user, { email: ldapUser.email, fallbackEmail: ldapUser.email, displayName: ldapUser.displayName, source: 'ldap' }, AuditSource.EXTERNAL_LDAP)); - if (userMappingError) debug('Failed to map user. %j %o', ldapUser, userMappingError); + if (userMappingError) log('Failed to map user. %j %o', ldapUser, userMappingError); } else if (user.email !== ldapUser.email || user.displayName !== ldapUser.displayName) { - debug(`syncUsers: [updating user] username=${ldapUser.username} email=${ldapUser.email} displayName=${ldapUser.displayName}`); + log(`syncUsers: [updating user] username=${ldapUser.username} email=${ldapUser.email} displayName=${ldapUser.displayName}`); const [userUpdateError] = await safe(users.update(user, { email: ldapUser.email, fallbackEmail: ldapUser.email, displayName: ldapUser.displayName }, AuditSource.EXTERNAL_LDAP)); - if (userUpdateError) debug('Failed to update user. %j %o', ldapUser, userUpdateError); + if (userUpdateError) log('Failed to update user. %j %o', ldapUser, userUpdateError); } else { // user known and up-to-date - debug(`syncUsers: [up-to-date user] username=${ldapUser.username} email=${ldapUser.email} displayName=${ldapUser.displayName}`); + log(`syncUsers: [up-to-date user] username=${ldapUser.username} email=${ldapUser.email} displayName=${ldapUser.displayName}`); } } - debug('syncUsers: done'); + log('syncUsers: done'); } async function syncGroups(config, progressCallback) { @@ -397,15 +397,15 @@ async function syncGroups(config, progressCallback) { assert.strictEqual(typeof progressCallback, 'function'); if (!config.syncGroups) { - debug('syncGroups: Group sync is disabled'); + log('syncGroups: Group sync is disabled'); progressCallback({ percent: 70, message: 'Skipping group sync' }); return []; } const ldapGroups = await ldapGroupSearch(config, {}); - debug(`syncGroups: Found ${ldapGroups.length} groups:`); - debug(ldapGroups); + log(`syncGroups: Found ${ldapGroups.length} groups:`); + log(ldapGroups); let percent = 40; const step = 28 / (ldapGroups.length + 1); @@ -423,19 +423,19 @@ async function syncGroups(config, progressCallback) { const result = await groups.getByName(groupName); if (!result) { - debug(`syncGroups: [adding group] groupname=${groupName}`); + log(`syncGroups: [adding group] groupname=${groupName}`); const [error] = await safe(groups.add({ name: groupName, source: 'ldap' }, AuditSource.EXTERNAL_LDAP)); - if (error) debug('syncGroups: Failed to create group', groupName, error); + if (error) log('syncGroups: Failed to create group', groupName, error); } else { // convert local group to ldap group. 2 reasons: // 1. we reset source flag when externalldap is disabled. if we renable, it automatically converts // 2. externalldap connector usually implies user wants to user external users/groups. groups.update(result.id, { source: 'ldap' }); - debug(`syncGroups: [up-to-date group] groupname=${groupName}`); + log(`syncGroups: [up-to-date group] groupname=${groupName}`); } } - debug('syncGroups: sync done'); + log('syncGroups: sync done'); } async function syncGroupMembers(config, progressCallback) { @@ -443,14 +443,14 @@ async function syncGroupMembers(config, progressCallback) { assert.strictEqual(typeof progressCallback, 'function'); if (!config.syncGroups) { - debug('syncGroupMembers: Group users sync is disabled'); + log('syncGroupMembers: Group users sync is disabled'); progressCallback({ percent: 98, message: 'Skipping group member sync' }); return []; } const allGroups = await groups.listWithMembers(); const ldapGroups = allGroups.filter(function (g) { return g.source === 'ldap'; }); - debug(`syncGroupMembers: Found ${ldapGroups.length} groups to sync users`); + log(`syncGroupMembers: Found ${ldapGroups.length} groups to sync users`); let percent = 70; const step = 28 / (ldapGroups.length + 1); @@ -458,11 +458,11 @@ async function syncGroupMembers(config, progressCallback) { for (const ldapGroup of ldapGroups) { percent = Math.min(percent + step, 98); progressCallback({ percent: Math.round(percent), message: `Syncing members of ${ldapGroup.name}` }); - debug(`syncGroupMembers: Sync users for group ${ldapGroup.name}`); + log(`syncGroupMembers: Sync users for group ${ldapGroup.name}`); const result = await ldapGroupSearch(config, {}); if (!result || result.length === 0) { - debug(`syncGroupMembers: Unable to find group ${ldapGroup.name} ignoring for now.`); + log(`syncGroupMembers: Unable to find group ${ldapGroup.name} ignoring for now.`); continue; } @@ -473,7 +473,7 @@ async function syncGroupMembers(config, progressCallback) { }); if (!found) { - debug(`syncGroupMembers: Unable to find group ${ldapGroup.name} ignoring for now.`); + log(`syncGroupMembers: Unable to find group ${ldapGroup.name} ignoring for now.`); continue; } @@ -482,24 +482,24 @@ async function syncGroupMembers(config, progressCallback) { // if only one entry is in the group ldap returns a string, not an array! if (typeof ldapGroupMembers === 'string') ldapGroupMembers = [ ldapGroupMembers ]; - debug(`syncGroupMembers: Group ${ldapGroup.name} has ${ldapGroupMembers.length} members.`); + log(`syncGroupMembers: Group ${ldapGroup.name} has ${ldapGroupMembers.length} members.`); const userIds = []; for (const memberDn of ldapGroupMembers) { const [ldapError, memberResult] = await safe(ldapGetByDN(config, memberDn)); if (ldapError) { - debug(`syncGroupMembers: Group ${ldapGroup.name} failed to get ${memberDn}: %o`, ldapError); + log(`syncGroupMembers: Group ${ldapGroup.name} failed to get ${memberDn}: %o`, ldapError); continue; } - debug(`syncGroupMembers: Group ${ldapGroup.name} has member object ${memberDn}`); + log(`syncGroupMembers: Group ${ldapGroup.name} has member object ${memberDn}`); const username = memberResult[config.usernameField]?.toLowerCase(); if (!username) continue; const [getError, userObject] = await safe(users.getByUsername(username)); if (getError || !userObject) { - debug(`syncGroupMembers: Failed to get user by username ${username}. %o`, getError ? getError : 'User not found'); + log(`syncGroupMembers: Failed to get user by username ${username}. %o`, getError ? getError : 'User not found'); continue; } @@ -507,15 +507,15 @@ async function syncGroupMembers(config, progressCallback) { } const membersChanged = ldapGroup.userIds.length !== userIds.length || ldapGroup.userIds.some(id => !userIds.includes(id)); if (membersChanged) { - debug(`syncGroupMembers: Group ${ldapGroup.name} changed.`); + log(`syncGroupMembers: Group ${ldapGroup.name} changed.`); const [setError] = await safe(groups.setMembers(ldapGroup, userIds, { skipSourceCheck: true }, AuditSource.EXTERNAL_LDAP)); - if (setError) debug(`syncGroupMembers: Failed to set members of group ${ldapGroup.name}. %o`, setError); + if (setError) log(`syncGroupMembers: Failed to set members of group ${ldapGroup.name}. %o`, setError); } else { - debug(`syncGroupMembers: Group ${ldapGroup.name} is unchanged.`); + log(`syncGroupMembers: Group ${ldapGroup.name} is unchanged.`); } } - debug('syncGroupMembers: done'); + log('syncGroupMembers: done'); } async function sync(progressCallback) { @@ -532,7 +532,7 @@ async function sync(progressCallback) { progressCallback({ percent: 100, message: 'Done' }); - debug('sync: done'); + log('sync: done'); } export default { diff --git a/src/hush.js b/src/hush.js index 5372dd841..243b623a9 100644 --- a/src/hush.js +++ b/src/hush.js @@ -1,10 +1,10 @@ import assert from 'node:assert'; import BoxError from './boxerror.js'; import crypto from 'node:crypto'; -import debugModule from 'debug'; +import logger from './logger.js'; import { Transform as TransformStream } from 'node:stream'; -const debug = debugModule('box:hush'); +const { log, trace } = logger('hush'); class EncryptStream extends TransformStream { constructor(encryption) { @@ -144,7 +144,7 @@ function decryptFilePath(filePath, encryption) { decryptedParts.push(plainTextString); } catch (error) { - debug(`Error decrypting part ${part} of path ${filePath}: %o`, error); + log(`Error decrypting part ${part} of path ${filePath}: %o`, error); return { error: new BoxError(BoxError.CRYPTO_ERROR, `Decryption error. ${part} of path ${filePath}: ${error.message}`) }; } } diff --git a/src/janitor.js b/src/janitor.js index 925594945..78fe13883 100644 --- a/src/janitor.js +++ b/src/janitor.js @@ -1,22 +1,22 @@ import assert from 'node:assert'; import BoxError from './boxerror.js'; -import debugModule from 'debug'; +import logger from './logger.js'; import Docker from 'dockerode'; import safe from 'safetydance'; import tokens from './tokens.js'; -const debug = debugModule('box:janitor'); +const { log, trace } = logger('janitor'); const gConnection = new Docker({ socketPath: '/var/run/docker.sock' }); async function cleanupTokens() { - debug('Cleaning up expired tokens'); + log('Cleaning up expired tokens'); const [error, result] = await safe(tokens.delExpired()); - if (error) return debug('cleanupTokens: error removing expired tokens. %o', error); + if (error) return log('cleanupTokens: error removing expired tokens. %o', error); - debug(`Cleaned up ${result} expired tokens`); + log(`Cleaned up ${result} expired tokens`); } async function cleanupTmpVolume(containerInfo) { @@ -24,7 +24,7 @@ async function cleanupTmpVolume(containerInfo) { const cmd = 'find /tmp -type f -mtime +10 -exec rm -rf {} +'.split(' '); // 10 day old files - debug(`cleanupTmpVolume ${JSON.stringify(containerInfo.Names)}`); + log(`cleanupTmpVolume ${JSON.stringify(containerInfo.Names)}`); const [error, execContainer] = await safe(gConnection.getContainer(containerInfo.Id).exec({ Cmd: cmd, AttachStdout: true, AttachStderr: true, Tty: false })); if (error) throw new BoxError(BoxError.DOCKER_ERROR, `Failed to exec container: ${error.message}`); @@ -41,16 +41,16 @@ async function cleanupTmpVolume(containerInfo) { } async function cleanupDockerVolumes() { - debug('Cleaning up docker volumes'); + log('Cleaning up docker volumes'); const [error, containers] = await safe(gConnection.listContainers({ all: 0 })); if (error) throw new BoxError(BoxError.DOCKER_ERROR, error); for (const container of containers) { - await safe(cleanupTmpVolume(container), { debug }); // intentionally ignore error + await safe(cleanupTmpVolume(container), { debug: log }); // intentionally ignore error } - debug('Cleaned up docker volumes'); + log('Cleaned up docker volumes'); } export default { diff --git a/src/ldapserver.js b/src/ldapserver.js index 88262b717..222ba855e 100644 --- a/src/ldapserver.js +++ b/src/ldapserver.js @@ -4,7 +4,7 @@ import apps from './apps.js'; import AuditSource from './auditsource.js'; import BoxError from './boxerror.js'; import constants from './constants.js'; -import debugModule from 'debug'; +import logger from './logger.js'; import eventlog from './eventlog.js'; import groups from './groups.js'; import ldap from 'ldapjs'; @@ -13,7 +13,7 @@ import safe from 'safetydance'; import users from './users.js'; import util from 'node:util'; -const debug = debugModule('box:ldapserver'); +const { log, trace } = logger('ldapserver'); let _MOCK_APP = null; @@ -142,7 +142,7 @@ function finalSend(results, req, res, next) { } async function userSearch(req, res, next) { - debug('user search: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id); + log('user search: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id); const [error, result] = await safe(getUsersWithAccessToApp(req)); if (error) return next(new ldap.OperationsError(error.message)); @@ -195,7 +195,7 @@ async function userSearch(req, res, next) { } async function groupSearch(req, res, next) { - debug('group search: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id); + log('group search: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id); const results = []; @@ -230,7 +230,7 @@ async function groupSearch(req, res, next) { } async function groupUsersCompare(req, res, next) { - debug('group users compare: dn %s, attribute %s, value %s (from %s)', req.dn.toString(), req.attribute, req.value, req.connection.ldap.id); + log('group users compare: dn %s, attribute %s, value %s (from %s)', req.dn.toString(), req.attribute, req.value, req.connection.ldap.id); const [error, result] = await safe(getUsersWithAccessToApp(req)); if (error) return next(new ldap.OperationsError(error.message)); @@ -245,7 +245,7 @@ async function groupUsersCompare(req, res, next) { } async function groupAdminsCompare(req, res, next) { - debug('group admins compare: dn %s, attribute %s, value %s (from %s)', req.dn.toString(), req.attribute, req.value, req.connection.ldap.id); + log('group admins compare: dn %s, attribute %s, value %s (from %s)', req.dn.toString(), req.attribute, req.value, req.connection.ldap.id); const [error, result] = await safe(getUsersWithAccessToApp(req)); if (error) return next(new ldap.OperationsError(error.message)); @@ -260,7 +260,7 @@ async function groupAdminsCompare(req, res, next) { } async function mailboxSearch(req, res, next) { - debug('mailbox search: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id); + log('mailbox search: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id); // if cn is set OR filter is mail= we only search for one mailbox specifically let email, dn; @@ -350,7 +350,7 @@ async function mailboxSearch(req, res, next) { } async function mailAliasSearch(req, res, next) { - debug('mail alias get: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id); + log('mail alias get: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id); if (!req.dn.rdns[0].attrs.cn) return next(new ldap.NoSuchObjectError('Missing CN')); @@ -389,7 +389,7 @@ async function mailAliasSearch(req, res, next) { } async function mailingListSearch(req, res, next) { - debug('mailing list get: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id); + log('mailing list get: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id); if (!req.dn.rdns[0].attrs.cn) return next(new ldap.NoSuchObjectError('Missing CN')); @@ -433,7 +433,7 @@ async function mailingListSearch(req, res, next) { // Will attach req.user if successful async function authenticateUser(req, res, next) { - debug('user bind: %s (from %s)', req.dn.toString(), req.connection.ldap.id); + log('user bind: %s (from %s)', req.dn.toString(), req.connection.ldap.id); const appId = req.app.id; @@ -478,7 +478,7 @@ async function verifyMailboxPassword(mailbox, password) { } async function authenticateSftp(req, res, next) { - debug('sftp auth: %s (from %s)', req.dn.toString(), req.connection.ldap.id); + log('sftp auth: %s (from %s)', req.dn.toString(), req.connection.ldap.id); if (!req.dn.rdns[0].attrs.cn) return next(new ldap.NoSuchObjectError('Missing CN')); @@ -492,13 +492,13 @@ async function authenticateSftp(req, res, next) { const [verifyError] = await safe(users.verifyWithUsername(parts[0], req.credentials, app.id, { skipTotpCheck: true })); if (verifyError) return next(new ldap.InvalidCredentialsError(verifyError.message)); - debug('sftp auth: success'); + log('sftp auth: success'); res.end(); } async function userSearchSftp(req, res, next) { - debug('sftp user search: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id); + log('sftp user search: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id); if (req.filter.attribute !== 'username' || !req.filter.value) return next(new ldap.NoSuchObjectError()); @@ -556,7 +556,7 @@ async function verifyAppMailboxPassword(serviceId, username, password) { } async function authenticateService(serviceId, dn, req, res, next) { - debug(`authenticateService: ${req.dn.toString()} (from ${req.connection.ldap.id})`); + log(`authenticateService: ${req.dn.toString()} (from ${req.connection.ldap.id})`); if (!dn.rdns[0].attrs.cn) return next(new ldap.NoSuchObjectError(dn.toString())); @@ -608,7 +608,7 @@ async function authenticateMail(req, res, next) { // https://ldapwiki.com/wiki/RootDSE / RFC 4512 - ldapsearch -x -h "${CLOUDRON_LDAP_SERVER}" -p "${CLOUDRON_LDAP_PORT}" -b "" -s base // ldapjs seems to call this handler for everything when search === '' async function maybeRootDSE(req, res, next) { - debug(`maybeRootDSE: requested with scope:${req.scope} dn:${req.dn.toString()}`); + log(`maybeRootDSE: requested with scope:${req.scope} dn:${req.dn.toString()}`); if (req.scope !== 'base') return next(new ldap.NoSuchObjectError()); // per the spec, rootDSE search require base scope if (!req.dn || req.dn.toString() !== '') return next(new ldap.NoSuchObjectError()); @@ -633,16 +633,16 @@ async function start() { const logger = { trace: NOOP, debug: NOOP, - info: debug, - warn: debug, - error: debug, - fatal: debug + info: log, + warn: log, + error: log, + fatal: log }; gServer = ldap.createServer({ log: logger }); gServer.on('error', function (error) { - debug('start: server error. %o', error); + log('start: server error. %o', error); }); gServer.search('ou=users,dc=cloudron', authenticateApp, userSearch); @@ -670,14 +670,14 @@ async function start() { // this is the bind for addons (after bind, they might search and authenticate) gServer.bind('ou=addons,dc=cloudron', function(req, res /*, next */) { - debug('addons bind: %s', req.dn.toString()); // note: cn can be email or id + log('addons bind: %s', req.dn.toString()); // note: cn can be email or id res.end(); }); // this is the bind for apps (after bind, they might search and authenticate user) gServer.bind('ou=apps,dc=cloudron', function(req, res /*, next */) { // TODO: validate password - debug('application bind: %s', req.dn.toString()); + log('application bind: %s', req.dn.toString()); res.end(); }); @@ -693,7 +693,7 @@ async function start() { // just log that an attempt was made to unknown route, this helps a lot during app packaging gServer.use(function(req, res, next) { - debug('not handled: dn %s, scope %s, filter %s (from %s)', req.dn ? req.dn.toString() : '-', req.scope, req.filter ? req.filter.toString() : '-', req.connection.ldap.id); + log('not handled: dn %s, scope %s, filter %s (from %s)', req.dn ? req.dn.toString() : '-', req.scope, req.filter ? req.filter.toString() : '-', req.connection.ldap.id); return next(); }); diff --git a/src/locks.js b/src/locks.js index 5a3898a38..34f97e563 100644 --- a/src/locks.js +++ b/src/locks.js @@ -1,10 +1,10 @@ import assert from 'node:assert'; import BoxError from './boxerror.js'; import database from './database.js'; -import debugModule from 'debug'; +import logger from './logger.js'; import promiseRetry from './promise-retry.js'; -const debug = debugModule('box:locks'); +const { log, trace } = logger('locks'); const TYPE_APP_TASK_PREFIX = 'app_task_'; const TYPE_APP_BACKUP_PREFIX = 'app_backup_'; @@ -31,7 +31,7 @@ async function write(value) { const result = await database.query('UPDATE locks SET dataJson=?, version=version+1 WHERE id=? AND version=?', [ JSON.stringify(value.data), 'platform', value.version ]); if (result.affectedRows !== 1) throw new BoxError(BoxError.CONFLICT, 'Someone updated before we did'); - debug(`write: current locks: ${JSON.stringify(value.data)}`); + log(`write: current locks: ${JSON.stringify(value.data)}`); } function canAcquire(data, type) { @@ -59,58 +59,58 @@ function canAcquire(data, type) { async function acquire(type) { assert.strictEqual(typeof type, 'string'); - await promiseRetry({ times: Number.MAX_SAFE_INTEGER, interval: 100, debug, retry: (error) => error.reason === BoxError.CONFLICT }, async () => { + await promiseRetry({ times: Number.MAX_SAFE_INTEGER, interval: 100, debug: log, retry: (error) => error.reason === BoxError.CONFLICT }, async () => { const { version, data } = await read(); const error = canAcquire(data, type); if (error) throw error; data[type] = gTaskId; await write({ version, data }); - debug(`acquire: ${type}`); + log(`acquire: ${type}`); }); } async function wait(type) { assert.strictEqual(typeof type, 'string'); - await promiseRetry({ times: Number.MAX_SAFE_INTEGER, interval: 10000, debug }, async () => await acquire(type)); + await promiseRetry({ times: Number.MAX_SAFE_INTEGER, interval: 10000, debug: log }, async () => await acquire(type)); } async function release(type) { assert.strictEqual(typeof type, 'string'); - await promiseRetry({ times: Number.MAX_SAFE_INTEGER, interval: 100, debug, retry: (error) => error.reason === BoxError.CONFLICT }, async () => { + await promiseRetry({ times: Number.MAX_SAFE_INTEGER, interval: 100, debug: log, retry: (error) => error.reason === BoxError.CONFLICT }, async () => { const { version, data } = await read(); if (!(type in data)) throw new BoxError(BoxError.BAD_STATE, `Lock ${type} was never acquired`); if (data[type] !== gTaskId) throw new BoxError(BoxError.BAD_STATE, `Task ${gTaskId} attempted to release lock ${type} acquired by ${data[type]}`); delete data[type]; await write({ version, data }); - debug(`release: ${type}`); + log(`release: ${type}`); }); } async function releaseAll() { await database.query('DELETE FROM locks'); await database.query('INSERT INTO locks (id, dataJson) VALUES (?, ?)', [ 'platform', JSON.stringify({}) ]); - debug('releaseAll: all locks released'); + log('releaseAll: all locks released'); } // identify programming errors in tasks that forgot to clean up locks async function releaseByTaskId(taskId) { assert.strictEqual(typeof taskId, 'string'); - await promiseRetry({ times: Number.MAX_SAFE_INTEGER, interval: 100, debug, retry: (error) => error.reason === BoxError.CONFLICT }, async () => { + await promiseRetry({ times: Number.MAX_SAFE_INTEGER, interval: 100, debug: log, retry: (error) => error.reason === BoxError.CONFLICT }, async () => { const { version, data } = await read(); for (const type of Object.keys(data)) { if (data[type] === taskId) { - debug(`releaseByTaskId: task ${taskId} forgot to unlock ${type}`); + log(`releaseByTaskId: task ${taskId} forgot to unlock ${type}`); delete data[type]; } } await write({ version, data }); - debug(`releaseByTaskId: ${taskId}`); + log(`releaseByTaskId: ${taskId}`); }); } diff --git a/src/logger.js b/src/logger.js new file mode 100644 index 000000000..deeb16c26 --- /dev/null +++ b/src/logger.js @@ -0,0 +1,16 @@ +import util from 'node:util'; + +const TRACE_ENABLED = false; +const LOG_ENABLED = process.env.BOX_ENV !== 'test' || !!process.env.LOG; + +function output(namespace, args) { + const ts = new Date().toISOString(); + process.stdout.write(`${ts} ${namespace}: ${util.format(...args)}\n`); +} + +export default function logger(namespace) { + return { + log: LOG_ENABLED ? (...args) => output(namespace, args) : () => {}, + trace: TRACE_ENABLED ? (...args) => output(namespace, args) : () => {}, + }; +} diff --git a/src/logs.js b/src/logs.js index a02644238..dc35a5550 100644 --- a/src/logs.js +++ b/src/logs.js @@ -1,11 +1,11 @@ import assert from 'node:assert'; import child_process from 'node:child_process'; -import debugModule from 'debug'; +import logger from './logger.js'; import path from 'node:path'; import stream from 'node:stream'; import { StringDecoder } from 'node:string_decoder'; -const debug = debugModule('box:logs'); +const { log, trace } = logger('logs'); const TransformStream = stream.Transform; const LOGTAIL_CMD = path.join(import.meta.dirname, 'scripts/logtail.sh'); @@ -74,7 +74,7 @@ function tail(filePaths, options) { cp.terminate = () => { child_process.execFile('/usr/bin/sudo', [ KILL_CHILD_CMD, cp.pid, process.pid ], { encoding: 'utf8' }, (error, stdout, stderr) => { - if (error) debug(`tail: failed to kill children`, stdout, stderr); + if (error) log(`tail: failed to kill children`, stdout, stderr); }); }; diff --git a/src/mail.js b/src/mail.js index 9ec03db03..3d2dfed7e 100644 --- a/src/mail.js +++ b/src/mail.js @@ -2,7 +2,7 @@ import assert from 'node:assert'; import BoxError from './boxerror.js'; import constants from './constants.js'; import database from './database.js'; -import debugModule from 'debug'; +import logger from './logger.js'; import dig from './dig.js'; import dns from './dns.js'; import eventlog from './eventlog.js'; @@ -22,7 +22,7 @@ import superagent from '@cloudron/superagent'; import validator from './validator.js'; import _ from './underscore.js'; -const debug = debugModule('box:mail'); +const { log, trace } = logger('mail'); const shell = shellModule('mail'); const OWNERTYPE_USER = 'user'; @@ -524,14 +524,14 @@ async function checkRbl(type, mailDomain) { const [rblError, records] = await safe(dig.resolve(`${flippedIp}.${rblServer.dns}`, 'A', DNS_OPTIONS)); if (rblError || records.length === 0) continue; // not listed - debug(`checkRbl (${domain}) flippedIp: ${flippedIp} is in the blocklist of ${rblServer.dns}: ${JSON.stringify(records)}`); + log(`checkRbl (${domain}) flippedIp: ${flippedIp} is in the blocklist of ${rblServer.dns}: ${JSON.stringify(records)}`); const result = Object.assign({}, rblServer); const [error2, txtRecords] = await safe(dig.resolve(`${flippedIp}.${rblServer.dns}`, 'TXT', DNS_OPTIONS)); result.txtRecords = error2 || !txtRecords ? [] : txtRecords.map(x => x.join('')); - debug(`checkRbl (${domain}) error: ${error2?.message || null} txtRecords: ${JSON.stringify(txtRecords)}`); + log(`checkRbl (${domain}) error: ${error2?.message || null} txtRecords: ${JSON.stringify(txtRecords)}`); blockedServers.push(result); } @@ -573,11 +573,11 @@ async function getStatus(domain) { for (let i = 0; i < checks.length; i++) { const response = responses[i], check = checks[i]; if (response.status !== 'fulfilled') { - debug(`check ${check.what} was rejected. This is not expected. reason: ${response.reason}`); + log(`check ${check.what} was rejected. This is not expected. reason: ${response.reason}`); continue; } - if (response.value.message) debug(`${check.what} (${domain}): ${response.value.message}`); + if (response.value.message) log(`${check.what} (${domain}): ${response.value.message}`); safe.set(results, checks[i].what, response.value || {}); } @@ -627,7 +627,7 @@ async function txtRecordsWithSpf(domain, mailFqdn) { const txtRecords = await dns.getDnsRecords('', domain, 'TXT'); - debug('txtRecordsWithSpf: current txt records - %j', txtRecords); + log('txtRecordsWithSpf: current txt records - %j', txtRecords); let i, matches, validSpf; @@ -644,10 +644,10 @@ async function txtRecordsWithSpf(domain, mailFqdn) { if (!matches) { // no spf record was found, create one txtRecords.push('"v=spf1 a:' + mailFqdn + ' ~all"'); - debug('txtRecordsWithSpf: adding txt record'); + log('txtRecordsWithSpf: adding txt record'); } else { // just add ourself txtRecords[i] = matches[1] + ' a:' + mailFqdn + txtRecords[i].slice(matches[1].length); - debug('txtRecordsWithSpf: inserting txt record'); + log('txtRecordsWithSpf: inserting txt record'); } return txtRecords; @@ -657,7 +657,7 @@ async function upsertDnsRecords(domain, mailFqdn) { assert.strictEqual(typeof domain, 'string'); assert.strictEqual(typeof mailFqdn, 'string'); - debug(`upsertDnsRecords: updating mail dns records domain:${domain} mailFqdn:${mailFqdn}`); + log(`upsertDnsRecords: updating mail dns records domain:${domain} mailFqdn:${mailFqdn}`); const mailDomain = await getDomain(domain); if (!mailDomain) throw new BoxError(BoxError.NOT_FOUND, 'mail domain not found'); @@ -679,13 +679,13 @@ async function upsertDnsRecords(domain, mailFqdn) { const dmarcRecords = await dns.getDnsRecords('_dmarc', domain, 'TXT'); // only update dmarc if absent. this allows user to set email for reporting if (dmarcRecords.length === 0) records.push({ subdomain: '_dmarc', domain, type: 'TXT', values: [ '"v=DMARC1; p=reject; pct=100"' ] }); - debug(`upsertDnsRecords: updating ${domain} with ${records.length} records: ${JSON.stringify(records)}`); + log(`upsertDnsRecords: updating ${domain} with ${records.length} records: ${JSON.stringify(records)}`); for (const record of records) { await dns.upsertDnsRecords(record.subdomain, record.domain, record.type, record.values); } - debug(`upsertDnsRecords: records of ${domain} added`); + log(`upsertDnsRecords: records of ${domain} added`); } async function setDnsRecords(domain) { @@ -714,7 +714,7 @@ async function setMailFromValidation(domain, enabled) { await updateDomain(domain, { mailFromValidation: enabled }); - safe(mailServer.restart(), { debug }); // have to restart mail container since haraka cannot watch symlinked config files (mail.ini) + safe(mailServer.restart(), { debug: log }); // have to restart mail container since haraka cannot watch symlinked config files (mail.ini) } async function setBanner(domain, banner) { @@ -723,7 +723,7 @@ async function setBanner(domain, banner) { await updateDomain(domain, { banner }); - safe(mailServer.restart(), { debug }); + safe(mailServer.restart(), { debug: log }); } async function setCatchAllAddress(domain, addresses) { @@ -736,7 +736,7 @@ async function setCatchAllAddress(domain, addresses) { await updateDomain(domain, { catchAll: addresses }); - safe(mailServer.restart(), { debug }); // have to restart mail container since haraka cannot watch symlinked config files (mail.ini) + safe(mailServer.restart(), { debug: log }); // have to restart mail container since haraka cannot watch symlinked config files (mail.ini) } async function setMailRelay(domain, relay, options) { @@ -760,7 +760,7 @@ async function setMailRelay(domain, relay, options) { await updateDomain(domain, { relay }); - safe(mailServer.restart(), { debug }); + safe(mailServer.restart(), { debug: log }); } async function setMailEnabled(domain, enabled, auditSource) { @@ -977,7 +977,7 @@ async function delMailbox(name, domain, options, auditSource) { if (result.affectedRows === 0) throw new BoxError(BoxError.NOT_FOUND, 'Mailbox not found'); const [error] = await safe(removeSolrIndex(mailbox)); - if (error) debug(`delMailbox: failed to remove solr index: ${error.message}`); + if (error) log(`delMailbox: failed to remove solr index: ${error.message}`); await eventlog.add(eventlog.ACTION_MAIL_MAILBOX_REMOVE, auditSource, { name, domain }); } @@ -1190,7 +1190,7 @@ async function resolveMailingList(listName, listDomain) { const member =`${memberName}@${memberDomain}`; // cleaned up without any '+' subaddress if (visited.includes(member)) { - debug(`resolveMailingList: list ${listName}@${listDomain} has a recursion at member ${member}`); + log(`resolveMailingList: list ${listName}@${listDomain} has a recursion at member ${member}`); continue; } visited.push(member); diff --git a/src/mailer.js b/src/mailer.js index eaca0220e..6cd812d16 100644 --- a/src/mailer.js +++ b/src/mailer.js @@ -3,7 +3,7 @@ import BoxError from './boxerror.js'; import branding from './branding.js'; import constants from './constants.js'; import dashboard from './dashboard.js'; -import debugModule from 'debug'; +import logger from './logger.js'; import ejs from 'ejs'; import mailServer from './mailserver.js'; import nodemailer from 'nodemailer'; @@ -11,7 +11,7 @@ import path from 'node:path'; import safe from 'safetydance'; import translations from './translations.js'; -const debug = debugModule('box:mailer'); +const { log, trace } = logger('mailer'); const _mailQueue = []; // accumulate mails in test mode; @@ -63,7 +63,7 @@ async function sendMail(mailOptions) { const [error] = await safe(transport.sendMail(mailOptions)); if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, `Error sending Email "${mailOptions.subject}" to ${mailOptions.to}: ${error.message}`); - debug(`Email "${mailOptions.subject}" sent to ${mailOptions.to}`); + log(`Email "${mailOptions.subject}" sent to ${mailOptions.to}`); } function render(templateFile, params, translationAssets) { @@ -73,7 +73,7 @@ function render(templateFile, params, translationAssets) { let content = null; let raw = safe.fs.readFileSync(path.join(MAIL_TEMPLATES_DIR, templateFile), 'utf8'); if (raw === null) { - debug(`Error loading ${templateFile}`); + log(`Error loading ${templateFile}`); return ''; } @@ -82,7 +82,7 @@ function render(templateFile, params, translationAssets) { try { content = ejs.render(raw, params); } catch (e) { - debug(`Error rendering ${templateFile}`, e); + log(`Error rendering ${templateFile}`, e); } return content; diff --git a/src/mailserver.js b/src/mailserver.js index e427244d9..d7f20868e 100644 --- a/src/mailserver.js +++ b/src/mailserver.js @@ -1,7 +1,7 @@ import assert from 'node:assert'; import BoxError from './boxerror.js'; import constants from './constants.js'; -import debugModule from 'debug'; +import logger from './logger.js'; import dns from './dns.js'; import docker from './docker.js'; import domains from './domains.js'; @@ -22,7 +22,7 @@ import shellModule from './shell.js'; import tasks from './tasks.js'; import users from './users.js'; -const debug = debugModule('box:mailserver'); +const { log, trace } = logger('mailserver'); const shell = shellModule('mailserver'); const DEFAULT_MEMORY_LIMIT = 512 * 1024 * 1024; @@ -31,7 +31,7 @@ const DEFAULT_MEMORY_LIMIT = 512 * 1024 * 1024; async function createMailConfig(mailFqdn) { assert.strictEqual(typeof mailFqdn, 'string'); - debug(`createMailConfig: generating mail config with ${mailFqdn}`); + log(`createMailConfig: generating mail config with ${mailFqdn}`); const mailDomains = await mail.listDomains(); @@ -124,7 +124,7 @@ async function configureMail(mailFqdn, mailDomain, serviceConfig) { // if the 'yellowtent' user of OS and the 'cloudron' user of mail container don't match, the keys become inaccessible by mail code if (!safe.fs.chmodSync(mailKeyFilePath, 0o644)) throw new BoxError(BoxError.FS_ERROR, `Could not chmod key file: ${safe.error.message}`); - debug('configureMail: stopping and deleting previous mail container'); + log('configureMail: stopping and deleting previous mail container'); await docker.stopContainer('mail'); await docker.deleteContainer('mail'); @@ -156,7 +156,7 @@ async function configureMail(mailFqdn, mailDomain, serviceConfig) { --label isCloudronManaged=true \ ${readOnly} -v /run -v /tmp ${image} ${cmd}`; - debug('configureMail: starting mail container'); + log('configureMail: starting mail container'); await shell.bash(runCmd, { encoding: 'utf8' }); } @@ -172,7 +172,7 @@ async function restart() { const mailConfig = await services.getServiceConfig('mail'); const { domain, fqdn } = await getLocation(); - debug(`restart: restarting mail container with mailFqdn:${fqdn} mailDomain:${domain}`); + log(`restart: restarting mail container with mailFqdn:${fqdn} mailDomain:${domain}`); // NOTE: the email container has to be re-created. this is because some of the settings like solr config rely on starting with a clean /run state await locks.wait(locks.TYPE_MAIL_SERVER_RESTART); @@ -184,7 +184,7 @@ async function restart() { async function start(existingInfra) { assert.strictEqual(typeof existingInfra, 'object'); - debug('startMail: starting'); + log('startMail: starting'); await restart(); if (existingInfra.version !== 'none' && existingInfra.images.mail !== infra.images.mail) await docker.deleteImage(existingInfra.images.mail); @@ -194,11 +194,11 @@ async function restartIfActivated() { const activated = await users.isActivated(); if (!activated) { - debug('restartIfActivated: skipping restart of mail container since Cloudron is not activated yet'); + log('restartIfActivated: skipping restart of mail container since Cloudron is not activated yet'); return; // not provisioned yet, do not restart container after dns setup } - debug('restartIfActivated: restarting on activated'); + log('restartIfActivated: restarting on activated'); await restart(); } @@ -208,7 +208,7 @@ async function onDomainAdded(domain) { const { fqdn } = await getLocation(); if (!fqdn) return; // mail domain is not set yet (when provisioning) - debug(`onDomainAdded: configuring mail for added domain ${domain}`); + log(`onDomainAdded: configuring mail for added domain ${domain}`); await mail.upsertDnsRecords(domain, fqdn); await restartIfActivated(); } @@ -216,7 +216,7 @@ async function onDomainAdded(domain) { async function onDomainRemoved(domain) { assert.strictEqual(typeof domain, 'string'); - debug(`onDomainRemoved: configuring mail for removed domain ${domain}`); + log(`onDomainRemoved: configuring mail for removed domain ${domain}`); await restart(); } @@ -224,10 +224,10 @@ async function checkCertificate() { const certificate = await reverseProxy.getMailCertificate(); const cert = safe.fs.readFileSync(`${paths.MAIL_CONFIG_DIR}/tls_cert.pem`, { encoding: 'utf8' }); if (cert === certificate.cert) { - debug('checkCertificate: certificate has not changed'); + log('checkCertificate: certificate has not changed'); return; } - debug('checkCertificate: certificate has changed'); + log('checkCertificate: certificate has changed'); await restartIfActivated(); } @@ -289,7 +289,7 @@ async function startChangeLocation(subdomain, domain, auditSource) { .then(async () => { await platform.onMailServerLocationChanged(auditSource); }) - .catch((taskError) => debug(`startChangeLocation`, taskError)); + .catch((taskError) => log(`startChangeLocation`, taskError)); await eventlog.add(eventlog.ACTION_MAIL_LOCATION, auditSource, { subdomain, domain, taskId }); return taskId; diff --git a/src/metrics.js b/src/metrics.js index af4c687f9..1c3140c42 100644 --- a/src/metrics.js +++ b/src/metrics.js @@ -2,7 +2,7 @@ import apps from './apps.js'; import assert from 'node:assert'; import BoxError from './boxerror.js'; import constants from './constants.js'; -import debugModule from 'debug'; +import logger from './logger.js'; import docker from './docker.js'; import fs from 'node:fs'; import net from 'node:net'; @@ -15,7 +15,7 @@ import shellModule from './shell.js'; import superagent from '@cloudron/superagent'; import _ from './underscore.js'; -const debug = debugModule('box:metrics'); +const { log, trace } = logger('metrics'); const shell = shellModule('metrics'); @@ -163,7 +163,7 @@ async function readSystemMetrics() { } async function sendToGraphite() { - // debug('sendStatsToGraphite: collecting stats'); + // log('sendStatsToGraphite: collecting stats'); const result = await readSystemMetrics(); @@ -203,7 +203,7 @@ async function sendToGraphite() { }); client.on('error', (error) => { - debug(`Error sending data to graphite: ${error.message}`); + log(`Error sending data to graphite: ${error.message}`); resolve(); }); @@ -381,7 +381,7 @@ async function pipeContainerToMap(name, statsMap) { // we used to poll before instead of a stream. but docker caches metrics internally and rate logic has to handle dups const statsStream = await docker.getStats(name, { stream: true }); - statsStream.on('error', (error) => debug(error)); + statsStream.on('error', (error) => log(error)); statsStream.on('data', (data) => { const stats = JSON.parse(data.toString('utf8')); const metrics = translateContainerStatsSync(stats); @@ -478,7 +478,7 @@ async function getStream(options) { const INTERVAL_MSECS = 1000; intervalId = setInterval(async () => { - if (options.system) await safe(pipeSystemToMap(statsMap), { debug }); + if (options.system) await safe(pipeSystemToMap(statsMap), { debug: log }); const result = {}; const nowSecs = Date.now() / 1000; // to match graphite return value diff --git a/src/mounts.js b/src/mounts.js index f6c8d69e6..338d4acc7 100644 --- a/src/mounts.js +++ b/src/mounts.js @@ -1,7 +1,7 @@ import assert from 'node:assert'; import BoxError from './boxerror.js'; import constants from './constants.js'; -import debugModule from 'debug'; +import logger from './logger.js'; import ejs from 'ejs'; import fs from 'node:fs'; import path from 'node:path'; @@ -9,7 +9,7 @@ import paths from './paths.js'; import safe from 'safetydance'; import shellModule from './shell.js'; -const debug = debugModule('box:mounts'); +const { log, trace } = logger('mounts'); const shell = shellModule('mounts'); const MOUNT_TYPE_FILESYSTEM = 'filesystem'; @@ -163,7 +163,7 @@ async function removeMount(mount) { if (constants.TEST) return; - await safe(shell.sudo([ RM_MOUNT_CMD, hostPath ], {}), { debug }); // ignore any error + await safe(shell.sudo([ RM_MOUNT_CMD, hostPath ], {}), { debug: log }); // ignore any error if (mountType === MOUNT_TYPE_SSHFS) { const keyFilePath = path.join(paths.SSHFS_KEYS_DIR, `identity_file_${path.basename(hostPath)}`); diff --git a/src/network/generic.js b/src/network/generic.js index 3b83d9552..0bc0e7dea 100644 --- a/src/network/generic.js +++ b/src/network/generic.js @@ -1,11 +1,11 @@ import assert from 'node:assert'; import BoxError from '../boxerror.js'; import constants from '../constants.js'; -import debugModule from 'debug'; +import logger from '../logger.js'; import safe from 'safetydance'; import superagent from '@cloudron/superagent'; -const debug = debugModule('box:network/generic'); +const { log, trace } = logger('network/generic'); const gCache = { ipv4: {}, ipv6: {} }; // each has { timestamp, value, request } @@ -15,16 +15,16 @@ async function getIP(type) { gCache[type].value = null; // clear the obsolete value - debug(`getIP: querying ${url} to get ${type}`); + log(`getIP: querying ${url} to get ${type}`); const [networkError, response] = await safe(superagent.get(url).timeout(30 * 1000).retry(2).ok(() => true)); if (networkError || response.status !== 200) { - debug(`getIP: Error getting IP. ${networkError.message}`); + log(`getIP: Error getting IP. ${networkError.message}`); throw new BoxError(BoxError.EXTERNAL_ERROR, `Unable to detect ${type}. API server (${type}.api.cloudron.io) unreachable`); } if (!response.body?.ip) { - debug('get: Unexpected answer. No "ip" found in response body.', response.body); + log('get: Unexpected answer. No "ip" found in response body.', response.body); throw new BoxError(BoxError.EXTERNAL_ERROR, `Unable to detect ${type}. No IP found in response`); } diff --git a/src/network/network-interface.js b/src/network/network-interface.js index 0164228f9..df32006b5 100644 --- a/src/network/network-interface.js +++ b/src/network/network-interface.js @@ -1,10 +1,10 @@ import assert from 'node:assert'; import BoxError from '../boxerror.js'; -import debugModule from 'debug'; +import logger from '../logger.js'; import os from 'node:os'; import safe from 'safetydance'; -const debug = debugModule('box:network/network-interface'); +const { log, trace } = logger('network/network-interface'); async function getIPv4(config) { @@ -16,7 +16,7 @@ async function getIPv4(config) { const addresses = iface.filter(i => i.family === 'IPv4').map(i => i.address); if (addresses.length === 0) throw new BoxError(BoxError.NETWORK_ERROR, `${config.ifname} does not have any IPv4 address`); - if (addresses.length > 1) debug(`${config.ifname} has multiple ipv4 - ${JSON.stringify(addresses)}. choosing the first one.`); + if (addresses.length > 1) log(`${config.ifname} has multiple ipv4 - ${JSON.stringify(addresses)}. choosing the first one.`); return addresses[0]; } @@ -30,7 +30,7 @@ async function getIPv6(config) { const addresses = iface.filter(i => i.family === 'IPv6').map(i => i.address); if (addresses.length === 0) throw new BoxError(BoxError.NETWORK_ERROR, `${config.ifname} does not have any IPv6 address`); - if (addresses.length > 1) debug(`${config.ifname} has multiple ipv6 - ${JSON.stringify(addresses)}. choosing the first one.`); + if (addresses.length > 1) log(`${config.ifname} has multiple ipv6 - ${JSON.stringify(addresses)}. choosing the first one.`); return addresses[0]; } diff --git a/src/notifications.js b/src/notifications.js index b5e011b0e..1f36a3ff7 100644 --- a/src/notifications.js +++ b/src/notifications.js @@ -4,13 +4,13 @@ import BoxError from './boxerror.js'; import changelog from './changelog.js'; import dashboard from './dashboard.js'; import database from './database.js'; -import debugModule from 'debug'; +import logger from './logger.js'; import eventlog from './eventlog.js'; import mailer from './mailer.js'; import safe from 'safetydance'; import users from './users.js'; -const debug = debugModule('box:notifications'); +const { log, trace } = logger('notifications'); const TYPE_CLOUDRON_INSTALLED = 'cloudronInstalled'; const TYPE_CLOUDRON_UPDATED = 'cloudronUpdated'; @@ -44,7 +44,7 @@ async function add(type, title, message, data) { assert.strictEqual(typeof message, 'string'); assert.strictEqual(typeof data, 'object'); - debug(`add: ${type} ${title}`); + log(`add: ${type} ${title}`); const query = 'INSERT INTO notifications (type, title, message, acknowledged, eventId, context) VALUES (?, ?, ?, ?, ?, ?)'; const args = [ type, title, message, false, data?.eventId || null, data.context || '' ]; @@ -147,7 +147,7 @@ async function oomEvent(eventId, containerId, app, addonName, event) { const admins = await users.getAdmins(); for (const admin of admins) { if (admin.notificationConfig.includes(TYPE_APP_OOM)) { - await safe(mailer.oomEvent(admin.email, containerId, app, addonName, event), { debug }); + await safe(mailer.oomEvent(admin.email, containerId, app, addonName, event), { debug: log }); } } } @@ -159,7 +159,7 @@ async function appUp(eventId, app) { const admins = await users.getAdmins(); for (const admin of admins) { if (admin.notificationConfig.includes(TYPE_APP_UP)) { - await safe(mailer.appUp(admin.email, app), { debug }); + await safe(mailer.appUp(admin.email, app), { debug: log }); } } } @@ -171,7 +171,7 @@ async function appDown(eventId, app) { const admins = await users.getAdmins(); for (const admin of admins) { if (admin.notificationConfig.includes(TYPE_APP_DOWN)) { - await safe(mailer.appDown(admin.email, app), { debug }); + await safe(mailer.appDown(admin.email, app), { debug: log }); } } } @@ -222,7 +222,7 @@ async function boxUpdateError(eventId, errorMessage) { const admins = await users.getAdmins(); for (const admin of admins) { if (admin.notificationConfig.includes(TYPE_CLOUDRON_UPDATE_FAILED)) { - await safe(mailer.boxUpdateError(admin.email, errorMessage), { debug }); + await safe(mailer.boxUpdateError(admin.email, errorMessage), { debug: log }); } } } @@ -237,7 +237,7 @@ async function certificateRenewalError(eventId, fqdn, errorMessage) { const admins = await users.getAdmins(); for (const admin of admins) { if (admin.notificationConfig.includes(TYPE_CERTIFICATE_RENEWAL_FAILED)) { - await safe(mailer.certificateRenewalError(admin.email, fqdn, errorMessage), { debug }); + await safe(mailer.certificateRenewalError(admin.email, fqdn, errorMessage), { debug: log }); } } } @@ -253,7 +253,7 @@ async function backupFailed(eventId, taskId, errorMessage) { const superadmins = await users.getSuperadmins(); for (const superadmin of superadmins) { if (superadmin.notificationConfig.includes(TYPE_BACKUP_FAILED)) { - await safe(mailer.backupFailed(superadmin.email, errorMessage, `https://${dashboardFqdn}/logs.html?taskId=${taskId}`), { debug }); + await safe(mailer.backupFailed(superadmin.email, errorMessage, `https://${dashboardFqdn}/logs.html?taskId=${taskId}`), { debug: log }); } } } @@ -262,7 +262,7 @@ async function rebootRequired() { const admins = await users.getAdmins(); for (const admin of admins) { if (admin.notificationConfig.includes(TYPE_REBOOT)) { - await safe(mailer.rebootRequired(admin.email), { debug }); + await safe(mailer.rebootRequired(admin.email), { debug: log }); } } } @@ -273,7 +273,7 @@ async function lowDiskSpace(message) { const admins = await users.getAdmins(); for (const admin of admins) { if (admin.notificationConfig.includes(TYPE_DISK_SPACE)) { - await safe(mailer.lowDiskSpace(admin.email, message), { debug }); + await safe(mailer.lowDiskSpace(admin.email, message), { debug: log }); } } } diff --git a/src/oidcserver.js b/src/oidcserver.js index 2534c9c6e..792610e95 100644 --- a/src/oidcserver.js +++ b/src/oidcserver.js @@ -7,7 +7,7 @@ import branding from './branding.js'; import constants from './constants.js'; import crypto from 'node:crypto'; import dashboard from './dashboard.js'; -import debugModule from 'debug'; +import logger from './logger.js'; import dns from './dns.js'; import ejs from 'ejs'; import express from 'express'; @@ -33,7 +33,7 @@ import util from 'node:util'; import Provider from 'oidc-provider'; import mailpasswords from './mailpasswords.js'; -const debug = debugModule('box:oidcserver'); +const { log, trace } = logger('oidcserver'); // 1. Index.vue starts the OIDC flow by navigating to /openid/auth. Webadmin sets callback url to authcallback.html + implicit flow @@ -83,12 +83,12 @@ class StorageAdapter { } constructor(name) { - debug(`Creating OpenID storage adapter for ${name}`); + log(`Creating OpenID storage adapter for ${name}`); this.name = name; } async upsert(id, payload, expiresIn) { - debug(`[${this.name}] upsert: ${id}`); + log(`[${this.name}] upsert: ${id}`); const expiresAt = expiresIn ? new Date(Date.now() + (expiresIn * 1000)) : 0; @@ -102,7 +102,7 @@ class StorageAdapter { const [error] = await safe(tokens.add({ clientId: payload.clientId, identifier: user.id, expires, accessToken: id, allowedIpRanges: '' })); if (error) { - debug('Error adding access token', error); + log('Error adding access token', error); throw error; } } else { @@ -111,12 +111,12 @@ class StorageAdapter { } async find(id) { - debug(`[${this.name}] find: ${id}`); + log(`[${this.name}] find: ${id}`); if (this.name === 'Client') { const [error, client] = await safe(oidcClients.get(id)); if (error || !client) { - debug('find: error getting client', error); + log('find: error getting client', error); return null; } @@ -132,7 +132,7 @@ class StorageAdapter { if (client.appId) { const [appError, app] = await safe(apps.get(client.appId)); if (appError || !app) { - debug(`find: Unknown app for client with appId ${client.appId}`); + log(`find: Unknown app for client with appId ${client.appId}`); return null; } @@ -183,12 +183,12 @@ class StorageAdapter { } async findByUserCode(userCode) { - debug(`[${this.name}] FIXME findByUserCode userCode:${userCode}`); + log(`[${this.name}] FIXME findByUserCode userCode:${userCode}`); } // this is called only on Session store. there is a payload.uid async findByUid(uid) { - debug(`[${this.name}] findByUid: ${uid}`); + log(`[${this.name}] findByUid: ${uid}`); const data = await StorageAdapter.getData(this.name); for (const d in data) { @@ -199,19 +199,19 @@ class StorageAdapter { } async consume(id) { - debug(`[${this.name}] consume: ${id}`); + log(`[${this.name}] consume: ${id}`); await StorageAdapter.updateData(this.name, (data) => data[id].consumed = true); } async destroy(id) { - debug(`[${this.name}] destroy: ${id}`); + log(`[${this.name}] destroy: ${id}`); await StorageAdapter.updateData(this.name, (data) => delete data[id]); } async revokeByGrantId(grantId) { - debug(`[${this.name}] revokeByGrantId: ${grantId}`); + log(`[${this.name}] revokeByGrantId: ${grantId}`); await StorageAdapter.updateData(this.name, (data) => { for (const d in data) { @@ -256,7 +256,7 @@ async function consumeAuthCode(authCode) { // This exposed to run on a cron job async function cleanupExpired() { - debug('cleanupExpired'); + log('cleanupExpired'); const types = [ 'AuthorizationCode', 'AccessToken', 'Grant', 'Interaction', 'RefreshToken', 'Session' ]; for (const type of types) { @@ -282,7 +282,7 @@ async function renderError(error) { language: await settings.get(settings.LANGUAGE_KEY), }; - debug('renderError: %o', error); + log('renderError: %o', error); return ejs.render(TEMPLATE_ERROR, data); } @@ -351,7 +351,7 @@ async function interactionLogin(req, res, next) { const ip = req.headers['x-forwarded-for'] || req.socket.remoteAddress || null; const clientId = details.params.client_id; - debug(`interactionLogin: for OpenID client ${clientId} from ${ip}`); + log(`interactionLogin: for OpenID client ${clientId} from ${ip}`); if (req.body.autoLoginToken) { // auto login for first admin/owner if (typeof req.body.autoLoginToken !== 'string') return next(new HttpError(400, 'autoLoginToken must be string if provided')); @@ -394,10 +394,10 @@ async function interactionLogin(req, res, next) { if (userPasskeys.length > 0) { const [passkeyError] = await safe(passkeys.verifyAuthentication(user, passkeyResponse)); if (passkeyError) { - debug(`interactionLogin: passkey verification failed for ${username}: ${passkeyError.message}`); + log(`interactionLogin: passkey verification failed for ${username}: ${passkeyError.message}`); return next(new HttpError(401, 'Invalid passkey')); } - debug(`interactionLogin: passkey verified for ${username}`); + log(`interactionLogin: passkey verified for ${username}`); } } @@ -446,7 +446,7 @@ async function interactionConfirm(req, res, next) { if (detailsError) return next(new HttpError(detailsError.statusCode, detailsError.error_description)); const { grantId, uid, prompt: { name, details }, params, session: { accountId }, lastSubmission } = interactionDetails; - debug(`route interaction confirm post uid:${uid} prompt.name:${name} accountId:${accountId}`); + log(`route interaction confirm post uid:${uid} prompt.name:${name} accountId:${accountId}`); const client = await oidcClients.get(params.client_id); if (!client) return next(new Error('Client not found')); @@ -510,7 +510,7 @@ async function interactionConfirm(req, res, next) { const auditSource = AuditSource.fromOidcRequest(req); await eventlog.add(user.ghost ? eventlog.ACTION_USER_LOGIN_GHOST : eventlog.ACTION_USER_LOGIN, auditSource, { userId: user.id, user: users.removePrivateFields(user), appId: client.appId || null }); - await safe(users.notifyLoginLocation(user, ip, userAgent, auditSource), { debug }); + await safe(users.notifyLoginLocation(user, ip, userAgent, auditSource), { debug: log }); const result = { consent }; await gOidcProvider.interactionFinished(req, res, result, { mergeWithLastSubmission: true }); @@ -586,31 +586,31 @@ async function start() { let keyEdDsa = await blobs.getString(blobs.OIDC_KEY_EDDSA); if (!keyEdDsa) { - debug('Generating new OIDC EdDSA key'); + log('Generating new OIDC EdDSA key'); const { privateKey } = await jose.generateKeyPair('EdDSA', { extractable: true }); keyEdDsa = Object.assign(await jose.exportJWK(privateKey), { alg: 'EdDSA' }); // alg is optional, but wp requires it await blobs.setString(blobs.OIDC_KEY_EDDSA, JSON.stringify(keyEdDsa)); jwksKeys.push(keyEdDsa); } else { - debug('Using existing OIDC EdDSA key'); + log('Using existing OIDC EdDSA key'); jwksKeys.push(JSON.parse(keyEdDsa)); } let keyRs256 = await blobs.getString(blobs.OIDC_KEY_RS256); if (!keyRs256) { - debug('Generating new OIDC RS256 key'); + log('Generating new OIDC RS256 key'); const { privateKey } = await jose.generateKeyPair('RS256', { extractable: true }); keyRs256 = Object.assign(await jose.exportJWK(privateKey), { alg: 'RS256' }); // alg is optional, but wp requires it await blobs.setString(blobs.OIDC_KEY_RS256, JSON.stringify(keyRs256)); jwksKeys.push(keyRs256); } else { - debug('Using existing OIDC RS256 key'); + log('Using existing OIDC RS256 key'); jwksKeys.push(JSON.parse(keyRs256)); } let cookieSecret = await settings.get(settings.OIDC_COOKIE_SECRET_KEY); if (!cookieSecret) { - debug('Generating new cookie secret'); + log('Generating new cookie secret'); cookieSecret = crypto.randomBytes(256).toString('base64'); await settings.set(settings.OIDC_COOKIE_SECRET_KEY, cookieSecret); } @@ -725,7 +725,7 @@ async function start() { const { subdomain, domain } = await dashboard.getLocation(); const fqdn = dns.fqdn(subdomain, domain); - debug(`start: create provider for ${fqdn} at ${ROUTE_PREFIX}`); + log(`start: create provider for ${fqdn} at ${ROUTE_PREFIX}`); gOidcProvider = new Provider(`https://${fqdn}${ROUTE_PREFIX}`, configuration); diff --git a/src/once.js b/src/once.js index c48be8eff..8db1fc104 100644 --- a/src/once.js +++ b/src/once.js @@ -1,12 +1,12 @@ -import debugModule from 'debug'; +import logger from './logger.js'; -const debug = debugModule('box:once'); +const { log, trace } = logger('once'); // https://github.com/isaacs/once/blob/main/LICENSE (ISC) function once (fn) { const f = function () { if (f.called) { - debug(`${f.name} was already called, returning previous return value`); + log(`${f.name} was already called, returning previous return value`); return f.value; } f.called = true; diff --git a/src/openssl.js b/src/openssl.js index 6a81cf58f..15954a321 100644 --- a/src/openssl.js +++ b/src/openssl.js @@ -1,19 +1,19 @@ import assert from 'node:assert'; import BoxError from './boxerror.js'; import crypto from 'node:crypto'; -import debugModule from 'debug'; +import logger from './logger.js'; import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import safe from 'safetydance'; import shellModule from './shell.js'; -const debug = debugModule('box:openssl'); +const { log, trace } = logger('openssl'); const shell = shellModule('openssl'); async function generateKey(type) { - debug(`generateKey: generating new key for${type}`); + log(`generateKey: generating new key for${type}`); if (type === 'rsa4096') { return await shell.spawn('openssl', ['genrsa', '4096'], { encoding: 'utf8' }); @@ -63,7 +63,7 @@ async function createCsr(key, cn, altNames) { // while we pass the CN anyways, subjectAltName takes precedence const csrPem = await shell.spawn('openssl', ['req', '-new', '-key', keyFilePath, '-outform', 'PEM', '-subj', `/CN=${cn}`, '-config', opensslConfigFile], { encoding: 'utf8' }); await safe(fs.promises.rm(tmpdir, { recursive: true, force: true })); - debug(`createCsr: csr file created for ${cn}`); + log(`createCsr: csr file created for ${cn}`); return csrPem; // inspect with openssl req -text -noout -in hostname.csr -inform pem }; @@ -81,7 +81,7 @@ async function getCertificateDates(cert) { const notAfterDate = new Date(notAfter); const daysLeft = (notAfterDate - new Date())/(24 * 60 * 60 * 1000); - debug(`expiryDate: ${lines[2]} notBefore=${notBefore} notAfter=${notAfter} daysLeft=${daysLeft}`); + log(`expiryDate: ${lines[2]} notBefore=${notBefore} notAfter=${notAfter} daysLeft=${daysLeft}`); return { startDate: notBeforeDate, endDate: notAfterDate }; } @@ -106,7 +106,7 @@ async function generateCertificate(domain) { const opensslConf = safe.fs.readFileSync('/etc/ssl/openssl.cnf', 'utf8'); const cn = domain; - debug(`generateCertificate: domain=${domain} cn=${cn}`); + log(`generateCertificate: domain=${domain} cn=${cn}`); // SAN must contain all the domains since CN check is based on implementation if SAN is found. -checkhost also checks only SAN if present! const opensslConfWithSan = `${opensslConf}\n[SAN]\nsubjectAltName=DNS:${domain},DNS:*.${cn}\n`; @@ -169,7 +169,7 @@ async function generateDkimKey() { } async function generateDhparam() { - debug('generateDhparam: generating dhparams'); + log('generateDhparam: generating dhparams'); return await shell.spawn('openssl', ['dhparam', '-dsaparam', '2048'], { encoding: 'utf8' }); } diff --git a/src/passkeys.js b/src/passkeys.js index 277691c75..a0b4908c7 100644 --- a/src/passkeys.js +++ b/src/passkeys.js @@ -3,7 +3,7 @@ import BoxError from './boxerror.js'; import crypto from 'node:crypto'; import dashboard from './dashboard.js'; import database from './database.js'; -import debugModule from 'debug'; +import logger from './logger.js'; import safe from 'safetydance'; import { generateRegistrationOptions, @@ -13,7 +13,7 @@ import { } from '@simplewebauthn/server'; import _ from './underscore.js'; -const debug = debugModule('box:passkeys'); +const { log, trace } = logger('passkeys'); const PASSKEY_FIELDS = [ 'id', 'userId', 'credentialId', 'publicKey', 'counter', 'transports', 'name', 'creationTime', 'lastUsedTime' ].join(','); @@ -151,7 +151,7 @@ async function getRegistrationOptions(user) { storeChallenge(user.id, options.challenge); - debug(`getRegistrationOptions: generated for user ${user.id}`); + log(`getRegistrationOptions: generated for user ${user.id}`); return options; } @@ -179,7 +179,7 @@ async function verifyRegistration(user, response, name) { })); if (error) { - debug(`verifyRegistration: verification failed for user ${user.id}:`, error); + log(`verifyRegistration: verification failed for user ${user.id}:`, error); throw new BoxError(BoxError.INVALID_CREDENTIALS, 'Passkey verification failed'); } @@ -200,7 +200,7 @@ async function verifyRegistration(user, response, name) { name || 'Passkey' ); - debug(`verifyRegistration: passkey registered for user ${user.id}`); + log(`verifyRegistration: passkey registered for user ${user.id}`); return result; } @@ -224,7 +224,7 @@ async function getAuthenticationOptions(user) { storeChallenge(user.id, options.challenge); - debug(`getAuthenticationOptions: generated for user ${user.id}`); + log(`getAuthenticationOptions: generated for user ${user.id}`); return options; } @@ -243,12 +243,12 @@ async function verifyAuthentication(user, response) { const passkey = await getByCredentialId(credentialIdBase64url); if (!passkey) { - debug(`verifyAuthentication: passkey not found for credential ${credentialIdBase64url}`); + log(`verifyAuthentication: passkey not found for credential ${credentialIdBase64url}`); throw new BoxError(BoxError.NOT_FOUND, 'Passkey not found'); } if (passkey.userId !== user.id) { - debug(`verifyAuthentication: passkey belongs to different user`); + log(`verifyAuthentication: passkey belongs to different user`); throw new BoxError(BoxError.INVALID_CREDENTIALS, 'Passkey does not belong to this user'); } @@ -268,7 +268,7 @@ async function verifyAuthentication(user, response) { })); if (error) { - debug(`verifyAuthentication: verification failed for user ${user.id}:`, error); + log(`verifyAuthentication: verification failed for user ${user.id}:`, error); throw new BoxError(BoxError.INVALID_CREDENTIALS, 'Passkey verification failed'); } @@ -276,7 +276,7 @@ async function verifyAuthentication(user, response) { await updateCounter(passkey.id, verification.authenticationInfo.newCounter); - debug(`verifyAuthentication: passkey verified for user ${user.id}`); + log(`verifyAuthentication: passkey verified for user ${user.id}`); return { verified: true, passkeyId: passkey.id }; } diff --git a/src/platform.js b/src/platform.js index 9a05960c3..21bc324fe 100644 --- a/src/platform.js +++ b/src/platform.js @@ -7,7 +7,7 @@ import constants from './constants.js'; import cron from './cron.js'; import dashboard from './dashboard.js'; import database from './database.js'; -import debugModule from 'debug'; +import logger from './logger.js'; import dockerProxy from './dockerproxy.js'; import fs from 'node:fs'; import infra from './infra_version.js'; @@ -26,7 +26,7 @@ import users from './users.js'; import volumes from './volumes.js'; import _ from './underscore.js'; -const debug = debugModule('box:platform'); +const { log, trace } = logger('platform'); const shell = shellModule('platform'); @@ -37,14 +37,14 @@ function getStatus() { } async function pruneVolumes() { - debug('pruneVolumes: remove all unused local volumes'); + log('pruneVolumes: remove all unused local volumes'); const [error] = await safe(shell.spawn('docker', [ 'volume', 'prune', '--all', '--force' ], { encoding: 'utf8' })); - if (error) debug(`pruneVolumes: error pruning volumes: ${error.mesage}`); + if (error) log(`pruneVolumes: error pruning volumes: ${error.mesage}`); } async function createDockerNetwork() { - debug('createDockerNetwork: recreating docker network'); + log('createDockerNetwork: recreating docker network'); await shell.spawn('docker', ['network', 'rm', '-f', 'cloudron'], {}); // the --ipv6 option will work even in ipv6 is disabled. fd00 is IPv6 ULA @@ -53,13 +53,13 @@ async function createDockerNetwork() { } async function removeAllContainers() { - debug('removeAllContainers: removing all containers for infra upgrade'); + log('removeAllContainers: removing all containers for infra upgrade'); const output = await shell.spawn('docker', ['ps', '-qa', '--filter', 'label=isCloudronManaged'], { encoding: 'utf8' }); if (!output) return; for (const containerId of output.trim().split('\n')) { - debug(`removeAllContainers: stopping and removing ${containerId}`); + log(`removeAllContainers: stopping and removing ${containerId}`); await shell.spawn('docker', ['stop', containerId], { encoding: 'utf8' }); await shell.spawn('docker', ['rm', '-f', containerId], { encoding: 'utf8' }); } @@ -70,10 +70,10 @@ async function markApps(existingInfra, restoreOptions) { assert.strictEqual(typeof restoreOptions, 'object'); // { skipDnsSetup } if (existingInfra.version === 'none') { // cloudron is being restored from backup - debug('markApps: restoring apps'); + log('markApps: restoring apps'); await apps.restoreApps(await apps.list(), restoreOptions, AuditSource.PLATFORM); } else if (existingInfra.version !== infra.version) { - debug('markApps: reconfiguring apps'); + log('markApps: reconfiguring apps'); reverseProxy.removeAppConfigs(); // should we change the cert location, nginx will not start await apps.configureApps(await apps.list(), { scheduleNow: false }, AuditSource.PLATFORM); // we will schedule it when infra is ready } else { @@ -85,25 +85,25 @@ async function markApps(existingInfra, restoreOptions) { if (changedAddons.length) { // restart apps if docker image changes since the IP changes and any "persistent" connections fail - debug(`markApps: changedAddons: ${JSON.stringify(changedAddons)}`); + log(`markApps: changedAddons: ${JSON.stringify(changedAddons)}`); await apps.restartAppsUsingAddons(changedAddons, AuditSource.PLATFORM); } else { - debug('markApps: apps are already uptodate'); + log('markApps: apps are already uptodate'); } } } async function onInfraReady(infraChanged) { - debug(`onInfraReady: platform is ready. infra changed: ${infraChanged}`); + log(`onInfraReady: platform is ready. infra changed: ${infraChanged}`); gStatus.message = 'Ready'; gStatus.state = 'ready'; - if (infraChanged) await safe(pruneVolumes(), { debug }); // ignore error + if (infraChanged) await safe(pruneVolumes(), { debug: log }); // ignore error await apps.schedulePendingTasks(AuditSource.PLATFORM); await appTaskManager.start(); // only prune services on infra change (which starts services for upgrade) - if (infraChanged) safe(services.stopUnusedServices(), { debug }); + if (infraChanged) safe(services.stopUnusedServices(), { debug: log }); } async function startInfra(restoreOptions) { @@ -111,7 +111,7 @@ async function startInfra(restoreOptions) { if (constants.TEST && !process.env.TEST_CREATE_INFRA) return; - debug('startInfra: checking infrastructure'); + log('startInfra: checking infrastructure'); let existingInfra = { version: 'none' }; if (fs.existsSync(paths.INFRA_VERSION_FILE)) { @@ -121,13 +121,13 @@ async function startInfra(restoreOptions) { // short-circuit for the restart case if (_.isEqual(infra, existingInfra)) { - debug('startInfra: infra is uptodate at version %s', infra.version); - safe(services.applyServiceLimits(), { debug }); + log('startInfra: infra is uptodate at version %s', infra.version); + safe(services.applyServiceLimits(), { debug: log }); await onInfraReady(false /* !infraChanged */); return; } - debug(`startInfra: updating infrastructure from ${existingInfra.version} to ${infra.version}`); + log(`startInfra: updating infrastructure from ${existingInfra.version} to ${infra.version}`); for (let attempt = 0; attempt < 5; attempt++) { try { @@ -147,7 +147,7 @@ async function startInfra(restoreOptions) { // for some reason, mysql arbitrary restarts making startup tasks fail. this makes the box update stuck // LOST is when existing connection breaks. REFUSED is when new connection cannot connect at all const retry = error.reason === BoxError.DATABASE_ERROR && (error.code === 'PROTOCOL_CONNECTION_LOST' || error.code === 'ECONNREFUSED'); - debug(`startInfra: Failed to start services. retry=${retry} (attempt ${attempt}): ${error}`); + log(`startInfra: Failed to start services. retry=${retry} (attempt ${attempt}): ${error}`); if (!retry) { gStatus.message = `Failed to start services. ${error.stdout ?? ''} ${error.stderr ?? ''}`; gStatus.state = 'failed'; @@ -163,7 +163,7 @@ async function startInfra(restoreOptions) { async function onActivated(restoreOptions) { assert.strictEqual(typeof restoreOptions, 'object'); // { skipDnsSetup } - debug('onActivated: starting post activation services'); + log('onActivated: starting post activation services'); // Starting the infra after a user is available means: // 1. mail bounces can now be sent to the cloudron owner @@ -177,11 +177,11 @@ async function onActivated(restoreOptions) { if (!constants.TEST) await timers.setTimeout(30000); await reverseProxy.writeDefaultConfig({ activated :true }); - debug('onActivated: finished'); + log('onActivated: finished'); } async function onDeactivated() { - debug('onDeactivated: stopping post activation services'); + log('onDeactivated: stopping post activation services'); await cron.stopJobs(); await dockerProxy.stop(); @@ -189,7 +189,7 @@ async function onDeactivated() { } async function uninitialize() { - debug('uninitializing platform'); + log('uninitializing platform'); if (await users.isActivated()) await onDeactivated(); @@ -201,7 +201,7 @@ async function onDashboardLocationSet(subdomain, domain) { assert.strictEqual(typeof subdomain, 'string'); assert.strictEqual(typeof domain, 'string'); - await safe(reverseProxy.writeDashboardConfig(subdomain, domain), { debug }); // ok to fail if no disk space + await safe(reverseProxy.writeDashboardConfig(subdomain, domain), { debug: log }); // ok to fail if no disk space await oidcServer.stop(); await oidcServer.start(); @@ -210,7 +210,7 @@ async function onDashboardLocationSet(subdomain, domain) { } async function initialize() { - debug('initialize: start platform'); + log('initialize: start platform'); await database.initialize(); await tasks.stopAllTasks(); // when box code crashes, systemd will clean up the control-group but not the tasks @@ -225,13 +225,13 @@ async function initialize() { // we remove the config as a simple security measure to not expose IP <-> domain const activated = await users.isActivated(); if (!activated) { - debug('initialize: not activated. generating IP based redirection config'); - await safe(reverseProxy.writeDefaultConfig({ activated: false }), { debug }); // ok to fail if no disk space + log('initialize: not activated. generating IP based redirection config'); + await safe(reverseProxy.writeDefaultConfig({ activated: false }), { debug: log }); // ok to fail if no disk space } await updater.notifyBoxUpdate(); - if (await users.isActivated()) safe(onActivated({ skipDnsSetup: false }), { debug }); // run in background + if (await users.isActivated()) safe(onActivated({ skipDnsSetup: false }), { debug: log }); // run in background } async function onDashboardLocationChanged(auditSource) { @@ -239,9 +239,9 @@ async function onDashboardLocationChanged(auditSource) { // mark all apps to be reconfigured, all have ExtraHosts injected const [, installedApps] = await safe(apps.list()); - await safe(apps.configureApps(installedApps, { scheduleNow: true }, auditSource), { debug }); + await safe(apps.configureApps(installedApps, { scheduleNow: true }, auditSource), { debug: log }); - await safe(services.rebuildService('turn', auditSource), { debug }); // to update the realm variable + await safe(services.rebuildService('turn', auditSource), { debug: log }); // to update the realm variable } async function onMailServerLocationChanged(auditSource) { @@ -250,7 +250,7 @@ async function onMailServerLocationChanged(auditSource) { // mark apps using email addon to be reconfigured const [, installedApps] = await safe(apps.list()); const appsUsingEmail = installedApps.filter((a) => !!a.manifest.addons?.email || a.manifest.addons?.sendmail?.requiresValidCertificate); - await safe(apps.configureApps(appsUsingEmail, { scheduleNow: true }, auditSource), { debug }); + await safe(apps.configureApps(appsUsingEmail, { scheduleNow: true }, auditSource), { debug: log }); } async function onMailServerIncomingDomainsChanged(auditSource) { @@ -259,7 +259,7 @@ async function onMailServerIncomingDomainsChanged(auditSource) { // mark apps using email addon to be reconfigured const [, installedApps] = await safe(apps.list()); const appsUsingEmail = installedApps.filter((a) => !!a.manifest.addons?.email); - await safe(apps.configureApps(appsUsingEmail, { scheduleNow: true }, auditSource), { debug }); + await safe(apps.configureApps(appsUsingEmail, { scheduleNow: true }, auditSource), { debug: log }); } export default { diff --git a/src/provision.js b/src/provision.js index 616153628..ea80f7bfe 100644 --- a/src/provision.js +++ b/src/provision.js @@ -6,7 +6,7 @@ import backuptask from './backuptask.js'; import BoxError from './boxerror.js'; import dashboard from './dashboard.js'; import constants from './constants.js'; -import debugModule from 'debug'; +import logger from './logger.js'; import dns from './dns.js'; import domains from './domains.js'; import eventlog from './eventlog.js'; @@ -27,7 +27,7 @@ import users from './users.js'; import tld from 'tldjs'; import tokens from './tokens.js'; -const debug = debugModule('box:provision'); +const { log, trace } = logger('provision'); // we cannot use tasks since the tasks table gets overwritten when db is imported @@ -51,13 +51,13 @@ const gStatus = { }; function setProgress(task, message) { - debug(`setProgress: ${task} - ${message}`); + log(`setProgress: ${task} - ${message}`); gStatus[task].message = message; } async function ensureDhparams() { if (fs.existsSync(paths.DHPARAMS_FILE)) return; - debug('ensureDhparams: generating dhparams'); + log('ensureDhparams: generating dhparams'); const dhparams = await openssl.generateDhparam(); if (!safe.fs.writeFileSync(paths.DHPARAMS_FILE, dhparams)) throw new BoxError(BoxError.FS_ERROR, `Could not save dhparams.pem: ${safe.error.message}`); } @@ -75,7 +75,7 @@ async function setupTask(domain, auditSource) { const location = { subdomain: constants.DASHBOARD_SUBDOMAIN, domain }; try { - debug(`setupTask: subdomain ${location.subdomain} and domain ${location.domain}`); + log(`setupTask: subdomain ${location.subdomain} and domain ${location.domain}`); await dns.registerLocations([location], { overwriteDns: true }, (progress) => setProgress('setup', progress.message)); await dns.waitForLocations([location], (progress) => setProgress('setup', progress.message)); await reverseProxy.ensureCertificate(location, {}, auditSource); @@ -85,7 +85,7 @@ async function setupTask(domain, auditSource) { setProgress('setup', 'Done'), await eventlog.add(eventlog.ACTION_PROVISION, auditSource, {}); } catch (error) { - debug('setupTask: error. %o', error); + log('setupTask: error. %o', error); gStatus.setup.errorMessage = error.message; } @@ -111,7 +111,7 @@ async function setup(domainConfig, ipv4Config, ipv6Config, auditSource) { const domain = domainConfig.domain.toLowerCase(); const zoneName = domainConfig.zoneName ? domainConfig.zoneName : (tld.getDomain(domain) || domain); - debug(`setup: domain ${domain} and zone ${zoneName}`); + log(`setup: domain ${domain} and zone ${zoneName}`); const data = { zoneName: zoneName, @@ -127,9 +127,9 @@ async function setup(domainConfig, ipv4Config, ipv6Config, auditSource) { await network.setIPv4Config(ipv4Config); await network.setIPv6Config(ipv6Config); - safe(setupTask(domain, auditSource), { debug }); // now that args are validated run the task in the background + safe(setupTask(domain, auditSource), { debug: log }); // now that args are validated run the task in the background } catch (error) { - debug('setup: error. %o', error); + log('setup: error. %o', error); gStatus.setup.active = false; gStatus.setup.errorMessage = error.message; throw error; @@ -144,7 +144,7 @@ async function activate(username, password, email, displayName, ip, auditSource) assert.strictEqual(typeof ip, 'string'); assert.strictEqual(typeof auditSource, 'object'); - debug(`activate: user: ${username} email:${email}`); + log(`activate: user: ${username} email:${email}`); await appstore.registerCloudron3(); @@ -157,7 +157,7 @@ async function activate(username, password, email, displayName, ip, auditSource) await eventlog.add(eventlog.ACTION_ACTIVATE, auditSource, {}); - safe(platform.onActivated({ skipDnsSetup: false }), { debug }); // background + safe(platform.onActivated({ skipDnsSetup: false }), { debug: log }); // background return { userId: ownerId, @@ -215,11 +215,11 @@ async function restoreTask(backupSite, remotePath, ipv4Config, ipv6Config, optio await appstore.checkSubscription(); // never throws. worst case, user has to visit the Account view to refresh subscription info - safe(platform.onActivated({ skipDnsSetup: options.skipDnsSetup }), { debug }); // background + safe(platform.onActivated({ skipDnsSetup: options.skipDnsSetup }), { debug: log }); // background await backupSites.storageApi(backupSite).teardown(backupSite.config); } catch (error) { - debug('restoreTask: error. %o', error); + log('restoreTask: error. %o', error); gStatus.restore.errorMessage = error ? error.message : ''; } gStatus.restore.active = false; @@ -260,9 +260,9 @@ async function restore(backupConfig, remotePath, version, ipv4Config, ipv6Config const ipv6Error = await network.testIPv6Config(ipv6Config); if (ipv6Error) throw ipv6Error; - safe(restoreTask(backupSite, remotePath, ipv4Config, ipv6Config, options, auditSource), { debug }); // now that args are validated run the task in the background + safe(restoreTask(backupSite, remotePath, ipv4Config, ipv6Config, options, auditSource), { debug: log }); // now that args are validated run the task in the background } catch (error) { - debug('restore: error. %o', error); + log('restore: error. %o', error); gStatus.restore.active = false; gStatus.restore.errorMessage = error.message; throw error; diff --git a/src/proxyauth.js b/src/proxyauth.js index d75f7b250..d414cb662 100644 --- a/src/proxyauth.js +++ b/src/proxyauth.js @@ -3,7 +3,7 @@ import assert from 'node:assert'; import blobs from './blobs.js'; import constants from './constants.js'; import dashboard from './dashboard.js'; -import debugModule from 'debug'; +import logger from './logger.js'; import ejs from 'ejs'; import express from 'express'; import fs from 'node:fs'; @@ -21,7 +21,7 @@ import settings from './settings.js'; import users from './users.js'; import util from 'node:util'; -const debug = debugModule('box:proxyAuth'); +const { log, trace } = logger('proxyAuth'); // heavily inspired from https://gock.net/blog/2020/nginx-subrequest-authentication-server/ and https://github.com/andygock/auth-server @@ -36,7 +36,7 @@ function jwtVerify(req, res, next) { jwt.verify(token, gTokenSecret, function (error, decoded) { if (error) { - debug('jwtVerify: malformed token or bad signature', error.message); + log('jwtVerify: malformed token or bad signature', error.message); req.user = null; } else { req.user = decoded.user || null; @@ -160,7 +160,7 @@ async function login(req, res, next) { async function callback(req, res, next) { if (typeof req.query.code !== 'string') return next(new HttpError(400, 'missing query argument "code"')); - debug(`callback: with code ${req.query.code}`); + log(`callback: with code ${req.query.code}`); const username = await oidcServer.consumeAuthCode(req.query.code); if (!username) return next(new HttpError(400, 'invalid "code"')); @@ -237,7 +237,7 @@ async function start() { gTokenSecret = await blobs.getString(blobs.PROXY_AUTH_TOKEN_SECRET); if (!gTokenSecret) { - debug('start: generating new token secret'); + log('start: generating new token secret'); gTokenSecret = hat(64); await blobs.setString(blobs.PROXY_AUTH_TOKEN_SECRET, gTokenSecret); } diff --git a/src/reverseproxy.js b/src/reverseproxy.js index f0ec13cc9..d03a3297c 100644 --- a/src/reverseproxy.js +++ b/src/reverseproxy.js @@ -5,7 +5,7 @@ import blobs from './blobs.js'; import BoxError from './boxerror.js'; import constants from './constants.js'; import dashboard from './dashboard.js'; -import debugModule from 'debug'; +import logger from './logger.js'; import dns from './dns.js'; import docker from './docker.js'; import domains from './domains.js'; @@ -24,7 +24,7 @@ import settings from './settings.js'; import shellModule from './shell.js'; import tasks from './tasks.js'; -const debug = debugModule('box:reverseproxy'); +const { log, trace } = logger('reverseproxy'); const shell = shellModule('reverseproxy'); const NGINX_APPCONFIG_EJS = fs.readFileSync(import.meta.dirname + '/nginxconfig.ejs', { encoding: 'utf8' }); @@ -59,7 +59,7 @@ async function providerMatches(domainObject, cert) { const mismatch = issuerMismatch || wildcardMismatch; - debug(`providerMatches: subject=${subject} domain=${domain} issuer=${issuer} ` + log(`providerMatches: subject=${subject} domain=${domain} issuer=${issuer} ` + `wildcard=${isWildcardCert}/${wildcard} prod=${isLetsEncryptProd}/${prod} ` + `issuerMismatch=${issuerMismatch} wildcardMismatch=${wildcardMismatch} match=${!mismatch}`); @@ -110,7 +110,7 @@ async function needsRenewal(cert, renewalInfo, options) { let renew = false; if (now.getTime() >= rt.getTime()) renew = true; // renew immediately since now is in the past else if ((now.getTime() + (24*60*60*1000)) >= rt.getTime()) renew = true; // next cron run will be in the past - debug(`needsRenewal: ${renew}. ARI ${JSON.stringify(renewalInfo)}`); + log(`needsRenewal: ${renew}. ARI ${JSON.stringify(renewalInfo)}`); return renew; // can wait } @@ -122,7 +122,7 @@ async function needsRenewal(cert, renewalInfo, options) { isExpiring = (endDate - now) <= (30 * 24 * 60 * 60 * 1000); // expiring in a month } - debug(`needsRenewal: ${isExpiring}. force: ${!!options.forceRenewal}`); + log(`needsRenewal: ${isExpiring}. force: ${!!options.forceRenewal}`); return isExpiring; } @@ -185,7 +185,7 @@ async function setupTlsAddon(app) { for (const content of contents) { if (writeFileSync(`${certificateDir}/${content.filename}`, content.data)) ++changed; } - debug(`setupTlsAddon: ${changed} files changed`); + log(`setupTlsAddon: ${changed} files changed`); // clean up any certs of old locations const filenamesInUse = new Set(contents.map(c => c.filename)); @@ -196,7 +196,7 @@ async function setupTlsAddon(app) { safe.fs.unlinkSync(path.join(certificateDir, filename)); ++removed; } - debug(`setupTlsAddon: ${removed} files removed`); + log(`setupTlsAddon: ${removed} files removed`); if (changed || removed) await docker.restartContainer(app.id); } @@ -215,7 +215,7 @@ async function setFallbackCertificate(domain, certificate) { assert.strictEqual(typeof domain, 'string'); assert(certificate && typeof certificate === 'object'); - debug(`setFallbackCertificate: setting certs for domain ${domain}`); + log(`setFallbackCertificate: setting certs for domain ${domain}`); if (!safe.fs.writeFileSync(path.join(paths.NGINX_CERT_DIR, `${domain}.host.cert`), certificate.cert)) throw new BoxError(BoxError.FS_ERROR, safe.error.message); if (!safe.fs.writeFileSync(path.join(paths.NGINX_CERT_DIR, `${domain}.host.key`), certificate.key)) throw new BoxError(BoxError.FS_ERROR, safe.error.message); @@ -245,7 +245,7 @@ async function writeCertificate(location) { const certFilePath = path.join(paths.NGINX_CERT_DIR, `${domain}.host.cert`); const keyFilePath = path.join(paths.NGINX_CERT_DIR, `${domain}.host.key`); - debug(`writeCertificate: ${fqdn} will use fallback certs`); + log(`writeCertificate: ${fqdn} will use fallback certs`); writeFileSync(certFilePath, domainObject.fallbackCertificate.cert); writeFileSync(keyFilePath, domainObject.fallbackCertificate.key); @@ -257,7 +257,7 @@ async function writeCertificate(location) { let key = await blobs.getString(`${blobs.CERT_PREFIX}-${certName}.key`); if (!key || !cert) { // use fallback certs if we didn't manage to get acme certs - debug(`writeCertificate: ${fqdn} will use fallback certs because acme is missing`); + log(`writeCertificate: ${fqdn} will use fallback certs because acme is missing`); cert = domainObject.fallbackCertificate.cert; key = domainObject.fallbackCertificate.key; } @@ -276,7 +276,7 @@ async function getKey(certName) { const key = await blobs.getString(`${blobs.CERT_PREFIX}-${certName}.key`); if (key) return key; - debug(`ensureKey: generating new key for ${certName}`); + log(`ensureKey: generating new key for ${certName}`); // secp384r1 is same as prime256v1. openssl ecparam -list_curves. we used to use secp384r1 but it doesn't seem to be accepted by few mail servers return await openssl.generateKey('secp256r1'); }; @@ -287,13 +287,13 @@ async function getRenewalInfo(cert, certName) { if (Date.now() < (new Date(renewalInfo.valid)).getTime()) return renewalInfo; // still valid - debug(`getRenewalInfo: ${certName} refreshing`); + log(`getRenewalInfo: ${certName} refreshing`); const [error, result] = await safe(acme2.getRenewalInfo(cert, renewalInfo.url)); if (error) { - debug(`getRenewalInfo: ${certName} error getting renewal info`, error); + log(`getRenewalInfo: ${certName} error getting renewal info`, error); await blobs.del(`${blobs.CERT_PREFIX}-${certName}.renewal`); } else { - debug(`getRenewalInfo: ${certName} updated: ${JSON.stringify(result)}`); + log(`getRenewalInfo: ${certName} updated: ${JSON.stringify(result)}`); await blobs.setString(`${blobs.CERT_PREFIX}-${certName}.renewal`, JSON.stringify(result)); } @@ -311,12 +311,12 @@ async function ensureCertificate(location, options, auditSource) { const fqdn = dns.fqdn(location.subdomain, location.domain); if (location.certificate) { // user certificate - debug(`ensureCertificate: ${fqdn} will use user certs`); + log(`ensureCertificate: ${fqdn} will use user certs`); return; } if (domainObject.tlsConfig.provider === 'fallback') { - debug(`ensureCertificate: ${fqdn} will use fallback certs`); + log(`ensureCertificate: ${fqdn} will use fallback certs`); return; } @@ -330,13 +330,13 @@ async function ensureCertificate(location, options, auditSource) { const outdated = await needsRenewal(cert, renewalInfo, options); if (sameProvider && !outdated) { - debug(`ensureCertificate: ${fqdn} acme cert exists and is up to date`); + log(`ensureCertificate: ${fqdn} acme cert exists and is up to date`); return; } - debug(`ensureCertificate: ${fqdn} acme cert exists but provider mismatch or needs renewal`); + log(`ensureCertificate: ${fqdn} acme cert exists but provider mismatch or needs renewal`); } - debug(`ensureCertificate: ${fqdn} needs acme cert`); + log(`ensureCertificate: ${fqdn} needs acme cert`); const [error, result] = await safe(acme2.getCertificate(fqdn, domainObject, key)); if (!error) { @@ -346,7 +346,7 @@ async function ensureCertificate(location, options, auditSource) { await blobs.setString(`${blobs.CERT_PREFIX}-${certName}.renewal`, JSON.stringify(result.renewalInfo)); } - debug(`ensureCertificate: error: ${error?.message || 'null'}`); + log(`ensureCertificate: error: ${error?.message || 'null'}`); await safe(eventlog.add(eventlog.ACTION_CERTIFICATE_NEW, auditSource, { domain: fqdn, errorMessage: error?.message || '', renewalInfo: result?.renewalInfo || null })); } @@ -377,7 +377,7 @@ async function writeDashboardConfig(subdomain, domain) { assert.strictEqual(typeof subdomain, 'string'); assert.strictEqual(typeof domain, 'string'); - debug(`writeDashboardConfig: writing dashboard config for ${domain}`); + log(`writeDashboardConfig: writing dashboard config for ${domain}`); const dashboardFqdn = dns.fqdn(subdomain, domain); const location = { domain, fqdn: dashboardFqdn, certificate: null }; @@ -390,7 +390,7 @@ async function removeDashboardConfig(subdomain, domain) { assert.strictEqual(typeof subdomain, 'string'); assert.strictEqual(typeof domain, 'string'); - debug(`removeDashboardConfig: removing dashboard config of ${domain}`); + log(`removeDashboardConfig: removing dashboard config of ${domain}`); const vhost = dns.fqdn(subdomain, domain); const nginxConfigFilename = path.join(paths.NGINX_APPCONFIG_DIR, `dashboard/${vhost}.conf`); @@ -473,7 +473,7 @@ async function writeAppLocationNginxConfig(app, location, certificatePath) { const nginxConf = ejs.render(NGINX_APPCONFIG_EJS, data); const filename = path.join(paths.NGINX_APPCONFIG_DIR, app.id, `${fqdn.replace('*', '_')}.conf`); - debug(`writeAppLocationNginxConfig: writing config for "${fqdn}" to ${filename} with options ${JSON.stringify(data)}`); + log(`writeAppLocationNginxConfig: writing config for "${fqdn}" to ${filename} with options ${JSON.stringify(data)}`); writeFileSync(filename, nginxConf); } @@ -566,7 +566,7 @@ async function cleanupCerts(locations, auditSource, progressCallback) { if (removedCertNames.length) await safe(eventlog.add(eventlog.ACTION_CERTIFICATE_CLEANUP, auditSource, { domains: removedCertNames })); - debug('cleanupCerts: done'); + log('cleanupCerts: done'); } async function checkCerts(options, auditSource, progressCallback) { @@ -621,12 +621,12 @@ async function startRenewCerts(options, auditSource) { assert.strictEqual(typeof auditSource, 'object'); const taskId = await tasks.add(tasks.TASK_CHECK_CERTS, [ options, auditSource ]); - safe(tasks.startTask(taskId, {}), { debug }); // background + safe(tasks.startTask(taskId, {}), { debug: log }); // background return taskId; } function removeAppConfigs() { - debug('removeAppConfigs: removing app nginx configs'); + log('removeAppConfigs: removing app nginx configs'); // remove all configs which are not the default or current dashboard for (const entry of fs.readdirSync(paths.NGINX_APPCONFIG_DIR, { withFileTypes: true })) { @@ -649,7 +649,7 @@ async function writeDefaultConfig(options) { const keyFilePath = path.join(paths.NGINX_CERT_DIR, 'default.key'); if (!fs.existsSync(certFilePath) || !fs.existsSync(keyFilePath)) { - debug('writeDefaultConfig: create new cert'); + log('writeDefaultConfig: create new cert'); const cn = 'cloudron-' + (new Date()).toISOString(); // randomize date a bit to keep firefox happy @@ -672,7 +672,7 @@ async function writeDefaultConfig(options) { const nginxConf = ejs.render(NGINX_APPCONFIG_EJS, data); const nginxConfigFilename = path.join(paths.NGINX_APPCONFIG_DIR, constants.NGINX_DEFAULT_CONFIG_FILE_NAME); - debug(`writeDefaultConfig: writing configs for endpoint "${data.endpoint}"`); + log(`writeDefaultConfig: writing configs for endpoint "${data.endpoint}"`); if (!safe.fs.writeFileSync(nginxConfigFilename, nginxConf)) throw new BoxError(BoxError.FS_ERROR, safe.error); @@ -708,7 +708,7 @@ async function setTrustedIps(trustedIps) { } async function reprovision() { - debug('reprovision: restoring fallback certs and trusted ips'); + log('reprovision: restoring fallback certs and trusted ips'); const result = await domains.list(); diff --git a/src/routes/accesscontrol.js b/src/routes/accesscontrol.js index a6ad9a9d3..aab5b58dc 100644 --- a/src/routes/accesscontrol.js +++ b/src/routes/accesscontrol.js @@ -1,13 +1,13 @@ import apps from '../apps.js'; import assert from 'node:assert'; import BoxError from '../boxerror.js'; -import debugModule from 'debug'; +import logger from '../logger.js'; import { HttpError } from '@cloudron/connect-lastmile'; import safe from 'safetydance'; import tokens from '../tokens.js'; import users from '../users.js'; -const debug = debugModule('box:routes/accesscontrol'); +const { log, trace } = logger('routes/accesscontrol'); async function passwordAuth(req, res, next) { @@ -55,7 +55,7 @@ async function tokenAuth(req, res, next) { const user = await users.get(token.identifier); if (!user) return next(new HttpError(401, 'User not found')); if (!user.active) { - debug(`tokenAuth: ${user.username || user.id} is not active`); + log(`tokenAuth: ${user.username || user.id} is not active`); return next(new HttpError(401, 'User not active')); } diff --git a/src/routes/apps.js b/src/routes/apps.js index 01803f0e1..04559406e 100644 --- a/src/routes/apps.js +++ b/src/routes/apps.js @@ -8,7 +8,7 @@ import backupSites from '../backupsites.js'; import BoxError from '../boxerror.js'; import community from '../community.js'; import constants from '../constants.js'; -import debugModule from 'debug'; +import logger from '../logger.js'; import { HttpError } from '@cloudron/connect-lastmile'; import { HttpSuccess } from '@cloudron/connect-lastmile'; import metrics from '../metrics.js'; @@ -18,7 +18,7 @@ import users from '../users.js'; import { getImageContentType } from '../image-content-type.js'; import WebSocket from 'ws'; -const debug = debugModule('box:routes/apps'); +const { log, trace } = logger('routes/apps'); async function load(req, res, next) { @@ -829,7 +829,7 @@ async function startExecWebSocket(req, res, next) { duplexStream.on('end', function () { ws.close(); }); duplexStream.on('close', function () { ws.close(); }); duplexStream.on('error', function (streamError) { - debug('duplexStream error: %o', streamError); + log('duplexStream error: %o', streamError); }); duplexStream.on('data', function (data) { if (ws.readyState !== WebSocket.OPEN) return; @@ -837,7 +837,7 @@ async function startExecWebSocket(req, res, next) { }); ws.on('error', function (wsError) { - debug('websocket error: %o', wsError); + log('websocket error: %o', wsError); }); ws.on('message', function (msg) { duplexStream.write(msg); diff --git a/src/routes/auth.js b/src/routes/auth.js index 56cd2839c..0bf22c5ce 100644 --- a/src/routes/auth.js +++ b/src/routes/auth.js @@ -2,7 +2,7 @@ import assert from 'node:assert'; import AuditSource from '../auditsource.js'; import BoxError from '../boxerror.js'; import constants from '../constants.js'; -import debugModule from 'debug'; +import logger from '../logger.js'; import eventlog from '../eventlog.js'; import { HttpError } from '@cloudron/connect-lastmile'; import { HttpSuccess } from '@cloudron/connect-lastmile'; @@ -12,7 +12,7 @@ import speakeasy from 'speakeasy'; import tokens from '../tokens.js'; import users from '../users.js'; -const debug = debugModule('box:routes/cloudron'); +const { log, trace } = logger('routes/cloudron'); async function login(req, res, next) { @@ -32,7 +32,7 @@ async function login(req, res, next) { const auditSource = AuditSource.fromRequest(req); await eventlog.add(req.user.ghost ? eventlog.ACTION_USER_LOGIN_GHOST : eventlog.ACTION_USER_LOGIN, auditSource, { userId: req.user.id, user: users.removePrivateFields(req.user), type, appId: oidcClients.ID_CLI }); - await safe(users.notifyLoginLocation(req.user, ip, userAgent, auditSource), { debug }); + await safe(users.notifyLoginLocation(req.user, ip, userAgent, auditSource), { debug: log }); next(new HttpSuccess(200, token)); } diff --git a/src/routes/mailserver.js b/src/routes/mailserver.js index c19556dbf..a65c65e6d 100644 --- a/src/routes/mailserver.js +++ b/src/routes/mailserver.js @@ -1,7 +1,7 @@ import assert from 'node:assert'; import AuditSource from '../auditsource.js'; import BoxError from '../boxerror.js'; -import debugModule from 'debug'; +import logger from '../logger.js'; import http from 'node:http'; import { HttpError } from '@cloudron/connect-lastmile'; import { HttpSuccess } from '@cloudron/connect-lastmile'; @@ -9,7 +9,7 @@ import mailServer from '../mailserver.js'; import safe from 'safetydance'; import services from '../services.js'; -const debug = debugModule('box:routes/mailserver'); +const { log, trace } = logger('routes/mailserver'); async function proxyToMailContainer(port, pathname, req, res, next) { @@ -61,7 +61,7 @@ async function proxyAndRestart(req, res, next) { if (httpError) return next(httpError); // for success, the proxy already sent the response. do not proceed to connect-lastmile which will result in double headers - await safe(mailServer.restart(), { debug }); + await safe(mailServer.restart(), { debug: log }); }); } diff --git a/src/routes/test/common.js b/src/routes/test/common.js index f9c752d86..7ee91f0b5 100644 --- a/src/routes/test/common.js +++ b/src/routes/test/common.js @@ -1,7 +1,7 @@ import apps from '../../apps.js'; import appstore from '../../appstore.js'; import backupSites from '../../backupsites.js'; -import debugModule from 'debug'; +import logger from '../../logger.js'; import constants from '../../constants.js'; import database from '../../database.js'; import assert from 'node:assert/strict'; @@ -16,7 +16,7 @@ import tasks from '../../tasks.js'; import timers from 'timers/promises'; import tokens from '../../tokens.js'; -const debug = debugModule('box:test/common'); +const { log, trace } = logger('test/common'); const manifest = { 'id': 'io.cloudron.test', @@ -98,17 +98,17 @@ const serverUrl = `http://localhost:${constants.PORT}`; async function setupServer() { - debug('Setting up server'); + log('Setting up server'); await database.initialize(); await database._clear(); await appstore._setApiServerOrigin(mockApiServerOrigin); await oidcServer.stop(); await server.start(); - debug('Set up server complete'); + log('Set up server complete'); } async function setup() { - debug('Setting up'); + log('Setting up'); await setupServer(); @@ -165,15 +165,15 @@ async function setup() { await settings._set(settings.APPSTORE_API_TOKEN_KEY, appstoreToken); // appstore token - debug('Setup complete'); + log('Setup complete'); } async function cleanup() { - debug('Cleaning up'); + log('Cleaning up'); await server.stop(); await oidcServer.stop(); if (!nock.isActive()) nock.activate(); - debug('Cleaned up'); + log('Cleaned up'); } function clearMailQueue() { @@ -187,7 +187,7 @@ async function checkMails(number) { } async function waitForTask(taskId) { - debug(`Waiting for task: ${taskId}`); + log(`Waiting for task: ${taskId}`); for (let i = 0; i < 30; i++) { const result = await tasks.get(taskId); @@ -197,7 +197,7 @@ async function waitForTask(taskId) { throw new Error(`Task ${taskId} failed: ${result.error.message} - ${result.error.stack}`); } await timers.setTimeout(2000); - debug(`Waiting for task to ${taskId} finish`); + log(`Waiting for task to ${taskId} finish`); } throw new Error(`Task ${taskId} never finished`); } @@ -206,16 +206,16 @@ async function waitForAsyncTask(es) { return new Promise((resolve, reject) => { const messages = []; es.addEventListener('message', function (message) { - debug(`waitForAsyncTask: ${message.data}`); + log(`waitForAsyncTask: ${message.data}`); messages.push(JSON.parse(message.data)); if (messages[messages.length-1].type === 'done') { - debug('waitForAsyncTask: finished'); + log('waitForAsyncTask: finished'); es.close(); resolve(messages); } }); es.addEventListener('error', function (error) { - debug('waitForAsyncTask: errored', error); + log('waitForAsyncTask: errored', error); es.close(); const e = new Error(error.message); e.code = error.code; diff --git a/src/scheduler.js b/src/scheduler.js index cc9045a56..9e9c28b58 100644 --- a/src/scheduler.js +++ b/src/scheduler.js @@ -4,12 +4,12 @@ import BoxError from './boxerror.js'; import cloudron from './cloudron.js'; import constants from './constants.js'; import { CronJob } from 'cron'; -import debugModule from 'debug'; +import logger from './logger.js'; import docker from './docker.js'; import safe from 'safetydance'; import _ from './underscore.js'; -const debug = debugModule('box:scheduler'); +const { log, trace } = logger('scheduler'); const gState = {}; // appId -> { containerId, schedulerConfig (manifest+crontab), cronjobs } @@ -17,12 +17,12 @@ const gSuspendedAppIds = new Set(); // suspended because some apptask is running // TODO: this should probably also stop existing jobs to completely prevent race but the code is not re-entrant function suspendAppJobs(appId) { - debug(`suspendAppJobs: ${appId}`); + log(`suspendAppJobs: ${appId}`); gSuspendedAppIds.add(appId); } function resumeAppJobs(appId) { - debug(`resumeAppJobs: ${appId}`); + log(`resumeAppJobs: ${appId}`); gSuspendedAppIds.delete(appId); } @@ -44,7 +44,7 @@ async function runTask(appId, taskName) { if (!error && data?.State?.Running === true) { const jobStartTime = new Date(data.State.StartedAt); // iso 8601 if ((new Date() - jobStartTime) < JOB_MAX_TIME) return; - debug(`runTask: ${containerName} is running too long, restarting`); + log(`runTask: ${containerName} is running too long, restarting`); } await docker.restartContainer(containerName); @@ -65,10 +65,10 @@ async function createJobs(app, schedulerConfig) { // stopJobs only deletes jobs since previous sync. This means that when box code restarts, none of the containers // are removed. The deleteContainer here ensures we re-create the cron containers with the latest config await safe(docker.deleteContainer(containerName)); // ignore error - const [error] = await safe(docker.createSubcontainer(app, containerName, [ '/bin/sh', '-c', command ], {} /* options */), { debug }); + const [error] = await safe(docker.createSubcontainer(app, containerName, [ '/bin/sh', '-c', command ], {} /* options */), { debug: log }); if (error && error.reason !== BoxError.ALREADY_EXISTS) continue; - debug(`createJobs: ${taskName} (${app.fqdn}) will run in container ${containerName}`); + log(`createJobs: ${taskName} (${app.fqdn}) will run in container ${containerName}`); let cronTime; if (schedule === '@service') { @@ -82,7 +82,7 @@ async function createJobs(app, schedulerConfig) { cronTime, onTick: async () => { const [taskError] = await safe(runTask(appId, taskName)); // put the app id in closure, so we don't use the outdated app object by mistake - if (taskError) debug(`could not run task ${taskName} : ${taskError.message}`); + if (taskError) log(`could not run task ${taskName} : ${taskError.message}`); }, start: true, timeZone: tz @@ -105,15 +105,15 @@ async function deleteAppJobs(appId, appState) { const containerName = `${appId}-${taskName}`; const [error] = await safe(docker.deleteContainer(containerName)); - if (error) debug(`deleteAppJobs: failed to delete task container with name ${containerName} : ${error.message}`); + if (error) log(`deleteAppJobs: failed to delete task container with name ${containerName} : ${error.message}`); } } async function deleteJobs() { for (const appId of Object.keys(gState)) { - debug(`deleteJobs: removing jobs of ${appId}`); + log(`deleteJobs: removing jobs of ${appId}`); const [error] = await safe(deleteAppJobs(appId, gState[appId])); - if (error) debug(`deleteJobs: error stopping jobs of removed app ${appId}: ${error.message}`); + if (error) log(`deleteJobs: error stopping jobs of removed app ${appId}: ${error.message}`); delete gState[appId]; } } @@ -125,12 +125,12 @@ async function sync() { const allAppIds = allApps.map(app => app.id); const removedAppIds = _.difference(Object.keys(gState), allAppIds); - if (removedAppIds.length !== 0) debug(`sync: stopping jobs of removed apps ${JSON.stringify(removedAppIds)}`); + if (removedAppIds.length !== 0) log(`sync: stopping jobs of removed apps ${JSON.stringify(removedAppIds)}`); for (const appId of removedAppIds) { - debug(`sync: removing jobs of ${appId}`); + log(`sync: removing jobs of ${appId}`); const [error] = await safe(deleteAppJobs(appId, gState[appId])); - if (error) debug(`sync: error stopping jobs of removed app ${appId}: ${error.message}`); + if (error) log(`sync: error stopping jobs of removed app ${appId}: ${error.message}`); delete gState[appId]; } @@ -143,10 +143,10 @@ async function sync() { if (_.isEqual(appState.schedulerConfig, schedulerConfig) && appState.containerId === app.containerId) continue; // nothing changed } - debug(`sync: clearing jobs of ${app.id} (${app.fqdn})`); + log(`sync: clearing jobs of ${app.id} (${app.fqdn})`); const [error] = await safe(deleteAppJobs(app.id, appState)); - if (error) debug(`sync: error stopping jobs of ${app.id} : ${error.message}`); + if (error) log(`sync: error stopping jobs of ${app.id} : ${error.message}`); if (!schedulerConfig) { // updated app version removed scheduler addon delete gState[app.id]; diff --git a/src/scripts/backupupload.js b/src/scripts/backupupload.js index 69cd3cbb8..7a93c9b0a 100755 --- a/src/scripts/backupupload.js +++ b/src/scripts/backupupload.js @@ -2,10 +2,10 @@ import backuptask from '../backuptask.js'; import database from '../database.js'; -import debugModule from 'debug'; +import logger from '../logger.js'; import safe from 'safetydance'; -const debug = debugModule('box:backupupload'); +const { log, trace } = logger('backupupload'); // --check is used by run-tests to verify sudo access works. Caller must set BOX_ENV (e.g. BOX_ENV=test). if (process.argv[2] === '--check') { @@ -18,7 +18,7 @@ const remotePath = process.argv[2]; const format = process.argv[3]; const dataLayoutString = process.argv[4]; -debug(`Backing up ${dataLayoutString} to ${remotePath}`); +log(`Backing up ${dataLayoutString} to ${remotePath}`); process.on('SIGTERM', function () { process.exit(0); @@ -26,7 +26,7 @@ process.on('SIGTERM', function () { // this can happen when the backup task is terminated (not box code) process.on('disconnect', function () { - debug('parent process died'); + log('parent process died'); process.exit(0); }); @@ -45,7 +45,7 @@ function throttledProgressCallback(msecs) { await database.initialize(); const [uploadError, result] = await safe(backuptask.upload(remotePath, format, dataLayoutString, throttledProgressCallback(5000))); -debug('upload completed. error: %o', uploadError); +log('upload completed. error: %o', uploadError); process.send({ result, errorMessage: uploadError?.message }); diff --git a/src/scripts/starttask.sh b/src/scripts/starttask.sh index ef2b19174..d2450c9a9 100755 --- a/src/scripts/starttask.sh +++ b/src/scripts/starttask.sh @@ -45,7 +45,7 @@ options="-p TimeoutStopSec=10s -p MemoryMax=${memory_limit_mb}M -p OOMScoreAdjus # it seems systemd-run does not return the exit status of the process despite --wait but atleast it waits if ! systemd-run --unit "${service_name}" --wait --uid=${id} --gid=${id} \ -p TimeoutStopSec=2s -p MemoryMax=${memory_limit_mb}M -p OOMScoreAdjust=${oom_score_adjust} --nice "${nice}" \ - --setenv HOME=${HOME} --setenv USER=${SUDO_USER} --setenv DEBUG=box:* --setenv BOX_ENV=${BOX_ENV} --setenv NODE_ENV=production \ + --setenv HOME=${HOME} --setenv USER=${SUDO_USER} --setenv BOX_ENV=${BOX_ENV} --setenv NODE_ENV=production \ "${task_worker}" "${task_id}" "${logfile}"; then echo "Service ${service_name} failed to run" # this only happens if the path to task worker itself is wrong fi diff --git a/src/server.js b/src/server.js index 9dd4372c2..d79b9cb7c 100644 --- a/src/server.js +++ b/src/server.js @@ -1,7 +1,7 @@ import assert from 'node:assert'; import AuditSource from './auditsource.js'; import constants from './constants.js'; -import debugModule from 'debug'; +import logger from './logger.js'; import eventlog from './eventlog.js'; import express from 'express'; import http from 'node:http'; @@ -14,14 +14,14 @@ import users from './users.js'; import util from 'node:util'; import { WebSocketServer } from 'ws'; -const debug = debugModule('box:server'); +const { log, trace } = logger('server'); let gHttpServer = null; function notFoundHandler(req, res, next) { const cleanUrl = req.url.replace(/(access_token=)[^&]+/, '$1' + ''); - debug(`no such route: ${req.method} ${cleanUrl}`); + log(`no such route: ${req.method} ${cleanUrl}`); return next(new HttpError(404, 'No such route')); } @@ -507,9 +507,9 @@ async function initializeExpressSync() { async function start() { assert(gHttpServer === null, 'Server is already up and running.'); - debug('=========================================='); - debug(` Cloudron ${constants.VERSION} `); - debug('=========================================='); + log('=========================================='); + log(` Cloudron ${constants.VERSION} `); + log('=========================================='); await platform.initialize(); diff --git a/src/services.js b/src/services.js index cf07234f6..8c04a4b35 100644 --- a/src/services.js +++ b/src/services.js @@ -7,7 +7,7 @@ import branding from './branding.js'; import constants from './constants.js'; import crypto from 'node:crypto'; import dashboard from './dashboard.js'; -import debugModule from 'debug'; +import logger from './logger.js'; import dig from './dig.js'; import docker from './docker.js'; import eventlog from './eventlog.js'; @@ -31,7 +31,7 @@ import sftp from './sftp.js'; import shellModule from './shell.js'; import superagent from '@cloudron/superagent'; -const debug = debugModule('box:services'); +const { log, trace } = logger('services'); const shell = shellModule('services'); const SERVICE_STATUS_STARTING = 'starting'; @@ -169,7 +169,7 @@ async function startAppServices(app) { const [error] = await safe(APP_SERVICES[addon].start(instance)); // assume addons name is service name // error ignored because we don't want "start app" to error. use can fix it from Services - if (error) debug(`startAppServices: ${addon}:${instance}. %o`, error); + if (error) log(`startAppServices: ${addon}:${instance}. %o`, error); } } @@ -182,7 +182,7 @@ async function stopAppServices(app) { const [error] = await safe(APP_SERVICES[addon].stop(instance)); // assume addons name is service name // error ignored because we don't want "start app" to error. use can fix it from Services - if (error) debug(`stopAppServices: ${addon}:${instance}. %o`, error); + if (error) log(`stopAppServices: ${addon}:${instance}. %o`, error); } } @@ -190,11 +190,11 @@ async function waitForContainer(containerName, tokenEnvName) { assert.strictEqual(typeof containerName, 'string'); assert.strictEqual(typeof tokenEnvName, 'string'); - debug(`Waiting for ${containerName}`); + log(`Waiting for ${containerName}`); const result = await getContainerDetails(containerName, tokenEnvName); - await promiseRetry({ times: 20, interval: 15000, debug }, async () => { + await promiseRetry({ times: 20, interval: 15000, debug: log }, async () => { const [networkError, response] = await safe(superagent.get(`http://${result.ip}:3000/healthcheck?access_token=${result.token}`) .timeout(20000) .ok(() => true)); @@ -217,7 +217,7 @@ async function ensureServiceRunning(serviceName) { if (error) throw new BoxError(BoxError.ADDONS_ERROR, `${serviceName} container not found`); if (container.State?.Running) return; - debug(`ensureServiceRunning: starting ${serviceName}`); + log(`ensureServiceRunning: starting ${serviceName}`); await docker.startContainer(serviceName); if (tokenEnvNames[serviceName]) await waitForContainer(serviceName, tokenEnvNames[serviceName]); @@ -232,22 +232,22 @@ async function stopUnusedServices() { } } - debug(`stopUnusedServices: used addons - ${[...usedAddons]}`); + log(`stopUnusedServices: used addons - ${[...usedAddons]}`); for (const name of LAZY_SERVICES) { if (usedAddons.has(name)) continue; - debug(`stopUnusedServices: stopping ${name} (no apps use it)`); - await safe(docker.stopContainer(name), { debug }); + log(`stopUnusedServices: stopping ${name} (no apps use it)`); + await safe(docker.stopContainer(name), { debug: log }); } } async function exportDatabase(addon) { assert.strictEqual(typeof addon, 'string'); - debug(`exportDatabase: exporting ${addon}`); + log(`exportDatabase: exporting ${addon}`); if (fs.existsSync(path.join(paths.ADDON_CONFIG_DIR, `exported-${addon}`))) { - debug(`exportDatabase: already exported addon ${addon} in previous run`); + log(`exportDatabase: already exported addon ${addon} in previous run`); return; } @@ -257,12 +257,12 @@ async function exportDatabase(addon) { if (!app.manifest.addons || !(addon in app.manifest.addons)) continue; // app doesn't use the addon if (app.installationState === apps.ISTATE_ERROR) continue; // missing db causes crash in old app addon containers - debug(`exportDatabase: exporting addon ${addon} of app ${app.id}`); + log(`exportDatabase: exporting addon ${addon} of app ${app.id}`); // eslint-disable-next-line no-use-before-define -- circular: ADDONS references setup fns, setup fns call exportDatabase const [error] = await safe(ADDONS[addon].backup(app, app.manifest.addons[addon])); if (error) { - debug(`exportDatabase: error exporting ${addon} of app ${app.id}. %o`, error); + log(`exportDatabase: error exporting ${addon} of app ${app.id}. %o`, error); // for errored apps, we can ignore if export had an error if (app.installationState === apps.ISTATE_ERROR) continue; throw error; @@ -279,19 +279,19 @@ async function exportDatabase(addon) { async function importDatabase(addon) { assert.strictEqual(typeof addon, 'string'); - debug(`importDatabase: importing ${addon}`); + log(`importDatabase: importing ${addon}`); const allApps = await apps.list(); for (const app of allApps) { if (!app.manifest.addons || !(addon in app.manifest.addons)) continue; // app doesn't use the addon - debug(`importDatabase: importing addon ${addon} of app ${app.id}`); + log(`importDatabase: importing addon ${addon} of app ${app.id}`); const [error] = await safe(importAppDatabase(app, addon)); // eslint-disable-line no-use-before-define if (!error) continue; - debug(`importDatabase: error importing ${addon} of app ${app.id}. Marking as errored. %o`, error); + log(`importDatabase: error importing ${addon} of app ${app.id}. Marking as errored. %o`, error); // FIXME: there is no way to 'repair' if we are here. we need to make a separate apptask that re-imports db // not clear, if repair workflow should be part of addon or per-app await safe(apps.update(app.id, { installationState: apps.ISTATE_ERROR, error: { message: error.message } })); @@ -304,7 +304,7 @@ async function setupLocalStorage(app, options) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); - debug('setupLocalStorage'); + log('setupLocalStorage'); const volumeDataDir = await apps.getStorageDir(app); @@ -316,7 +316,7 @@ async function clearLocalStorage(app, options) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); - debug('clearLocalStorage'); + log('clearLocalStorage'); const volumeDataDir = await apps.getStorageDir(app); const [error] = await safe(shell.sudo([ CLEARVOLUME_CMD, volumeDataDir ], {})); @@ -327,7 +327,7 @@ async function teardownLocalStorage(app, options) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); - debug('teardownLocalStorage'); + log('teardownLocalStorage'); const volumeDataDir = await apps.getStorageDir(app); const [error] = await safe(shell.sudo([ RMVOLUME_CMD, volumeDataDir ], {})); @@ -340,7 +340,7 @@ async function backupSqlite(app, options) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); - debug('Backing up sqlite'); + log('Backing up sqlite'); const volumeDataDir = await apps.getStorageDir(app); @@ -372,7 +372,7 @@ async function restoreSqlite(app, options) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); - debug('Restoring sqlite'); + log('Restoring sqlite'); const volumeDataDir = await apps.getStorageDir(app); @@ -408,7 +408,7 @@ async function setupPersistentDirs(app) { if (!app.manifest.persistentDirs) return; - debug('setupPersistentDirs'); + log('setupPersistentDirs'); for (const dir of app.manifest.persistentDirs) { const hostDir = persistentDirHostPath(app.id, dir); @@ -422,7 +422,7 @@ async function teardownPersistentDirs(app) { if (!app.manifest.persistentDirs) return; - debug('teardownPersistentDirs'); + log('teardownPersistentDirs'); for (const dir of app.manifest.persistentDirs) { const hostDir = persistentDirHostPath(app.id, dir); @@ -438,7 +438,7 @@ async function runBackupCommand(app) { if (!app.manifest.backupCommand) return; - debug('runBackupCommand'); + log('runBackupCommand'); const volumeDataDir = await apps.getStorageDir(app); const dataDirMount = volumeDataDir ? `-v ${volumeDataDir}:/app/data` : ''; @@ -462,7 +462,7 @@ async function runRestoreCommand(app) { if (!app.manifest.restoreCommand) return; - debug('runRestoreCommand'); + log('runRestoreCommand'); const volumeDataDir = await apps.getStorageDir(app); const dataDirMount = volumeDataDir ? `-v ${volumeDataDir}:/app/data` : ''; @@ -504,7 +504,7 @@ async function setupTurn(app, options) { { name: 'CLOUDRON_TURN_SECRET', value: turnSecret } ]; - debug('Setting up TURN'); + log('Setting up TURN'); await addonConfigs.set(app.id, 'turn', env); } @@ -513,7 +513,7 @@ async function teardownTurn(app, options) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); - debug('Tearing down TURN'); + log('Tearing down TURN'); await addonConfigs.unset(app.id, 'turn'); } @@ -543,7 +543,7 @@ async function setupEmail(app, options) { { name: 'CLOUDRON_EMAIL_LDAP_MAILBOXES_BASE_DN', value: 'ou=mailboxes,dc=cloudron' } ]; - debug('Setting up Email'); + log('Setting up Email'); await addonConfigs.set(app.id, 'email', env); } @@ -552,7 +552,7 @@ async function teardownEmail(app, options) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); - debug('Tearing down Email'); + log('Tearing down Email'); await addonConfigs.unset(app.id, 'email'); } @@ -574,7 +574,7 @@ async function setupLdap(app, options) { { name: 'CLOUDRON_LDAP_BIND_PASSWORD', value: hat(4 * 128) } // this is ignored ]; - debug('Setting up LDAP'); + log('Setting up LDAP'); await addonConfigs.set(app.id, 'ldap', env); } @@ -583,7 +583,7 @@ async function teardownLdap(app, options) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); - debug('Tearing down LDAP'); + log('Tearing down LDAP'); await addonConfigs.unset(app.id, 'ldap'); } @@ -592,7 +592,7 @@ async function setupSendMail(app, options) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); - debug('Setting up SendMail'); + log('Setting up SendMail'); const disabled = app.manifest.addons.sendmail.optional && !app.enableMailbox; if (disabled) return await addonConfigs.unset(app.id, 'sendmail'); @@ -616,7 +616,7 @@ async function setupSendMail(app, options) { if (app.manifest.addons.sendmail.supportsDisplayName) env.push({ name: 'CLOUDRON_MAIL_FROM_DISPLAY_NAME', value: app.mailboxDisplayName }); - debug('Setting sendmail addon config to %j', env); + log('Setting sendmail addon config to %j', env); await addonConfigs.set(app.id, 'sendmail', env); } @@ -624,7 +624,7 @@ async function teardownSendMail(app, options) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); - debug('Tearing down sendmail'); + log('Tearing down sendmail'); await addonConfigs.unset(app.id, 'sendmail'); } @@ -633,7 +633,7 @@ async function setupRecvMail(app, options) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); - debug('setupRecvMail: setting up recvmail'); + log('setupRecvMail: setting up recvmail'); if (!app.enableInbox) return await addonConfigs.unset(app.id, 'recvmail'); @@ -653,7 +653,7 @@ async function setupRecvMail(app, options) { { name: 'CLOUDRON_MAIL_TO_DOMAIN', value: app.inboxDomain }, ]; - debug('setupRecvMail: setting recvmail addon config to %j', env); + log('setupRecvMail: setting recvmail addon config to %j', env); await addonConfigs.set(app.id, 'recvmail', env); } @@ -661,7 +661,7 @@ async function teardownRecvMail(app, options) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); - debug('teardownRecvMail: tearing down recvmail'); + log('teardownRecvMail: tearing down recvmail'); await addonConfigs.unset(app.id, 'recvmail'); } @@ -685,7 +685,7 @@ async function startMysql(existingInfra) { const upgrading = existingInfra.version !== 'none' && requiresUpgrade(existingInfra.images.mysql, image); if (upgrading) { - debug('startMysql: mysql will be upgraded'); + log('startMysql: mysql will be upgraded'); await exportDatabase('mysql'); } @@ -711,11 +711,11 @@ async function startMysql(existingInfra) { --cap-add SYS_NICE \ ${readOnly} -v /tmp -v /run ${image} ${cmd}`; - debug('startMysql: stopping and deleting previous mysql container'); + log('startMysql: stopping and deleting previous mysql container'); await docker.stopContainer('mysql'); await docker.deleteContainer('mysql'); - debug('startMysql: starting mysql container'); + log('startMysql: starting mysql container'); await shell.bash(runCmd, { encoding: 'utf8' }); if (!serviceConfig.recoveryMode) { @@ -730,7 +730,7 @@ async function setupMySql(app, options) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); - debug('Setting up mysql'); + log('Setting up mysql'); await ensureServiceRunning('mysql'); @@ -770,7 +770,7 @@ async function setupMySql(app, options) { ); } - debug('Setting mysql addon config to %j', env); + log('Setting mysql addon config to %j', env); await addonConfigs.set(app.id, 'mysql', env); } @@ -813,7 +813,7 @@ async function backupMySql(app, options) { const database = mysqlDatabaseName(app.id); - debug('Backing up mysql'); + log('Backing up mysql'); const result = await getContainerDetails('mysql', 'CLOUDRON_MYSQL_TOKEN'); @@ -829,7 +829,7 @@ async function restoreMySql(app, options) { const database = mysqlDatabaseName(app.id); - debug('restoreMySql'); + log('restoreMySql'); const result = await getContainerDetails('mysql', 'CLOUDRON_MYSQL_TOKEN'); @@ -855,7 +855,7 @@ async function startPostgresql(existingInfra) { const upgrading = existingInfra.version !== 'none' && requiresUpgrade(existingInfra.images.postgresql, image); if (upgrading) { - debug('startPostgresql: postgresql will be upgraded'); + log('startPostgresql: postgresql will be upgraded'); await exportDatabase('postgresql'); } @@ -880,11 +880,11 @@ async function startPostgresql(existingInfra) { --label isCloudronManaged=true \ ${readOnly} -v /tmp -v /run ${image} ${cmd}`; - debug('startPostgresql: stopping and deleting previous postgresql container'); + log('startPostgresql: stopping and deleting previous postgresql container'); await docker.stopContainer('postgresql'); await docker.deleteContainer('postgresql'); - debug('startPostgresql: starting postgresql container'); + log('startPostgresql: starting postgresql container'); await shell.bash(runCmd, { encoding: 'utf8' }); if (!serviceConfig.recoveryMode) { @@ -899,7 +899,7 @@ async function setupPostgreSql(app, options) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); - debug('Setting up postgresql'); + log('Setting up postgresql'); await ensureServiceRunning('postgresql'); @@ -931,7 +931,7 @@ async function setupPostgreSql(app, options) { { name: 'CLOUDRON_POSTGRESQL_DATABASE', value: data.database } ]; - debug('Setting postgresql addon config to %j', env); + log('Setting postgresql addon config to %j', env); await addonConfigs.set(app.id, 'postgresql', env); } @@ -942,7 +942,7 @@ async function clearPostgreSql(app, options) { const { database, username } = postgreSqlNames(app.id); const locale = options.locale || 'C'; - debug('Clearing postgresql'); + log('Clearing postgresql'); const result = await getContainerDetails('postgresql', 'CLOUDRON_POSTGRESQL_TOKEN'); @@ -972,7 +972,7 @@ async function backupPostgreSql(app, options) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); - debug('Backing up postgresql'); + log('Backing up postgresql'); const { database } = postgreSqlNames(app.id); @@ -986,7 +986,7 @@ async function restorePostgreSql(app, options) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); - debug('Restore postgresql'); + log('Restore postgresql'); const { database, username } = postgreSqlNames(app.id); @@ -1008,7 +1008,7 @@ async function startMongodb(existingInfra) { const upgrading = existingInfra.version !== 'none' && requiresUpgrade(existingInfra.images.mongodb, image); if (upgrading) { - debug('startMongodb: mongodb will be upgraded'); + log('startMongodb: mongodb will be upgraded'); await exportDatabase('mongodb'); } @@ -1032,16 +1032,16 @@ async function startMongodb(existingInfra) { --label isCloudronManaged=true \ ${readOnly} -v /tmp -v /run ${image} ${cmd}`; - debug('startMongodb: stopping and deleting previous mongodb container'); + log('startMongodb: stopping and deleting previous mongodb container'); await docker.stopContainer('mongodb'); await docker.deleteContainer('mongodb'); if (!await hasAVX()) { - debug('startMongodb: not starting mongodb because CPU does not have AVX'); + log('startMongodb: not starting mongodb because CPU does not have AVX'); return; } - debug('startMongodb: starting mongodb container'); + log('startMongodb: starting mongodb container'); await shell.bash(runCmd, { encoding: 'utf8' }); if (!serviceConfig.recoveryMode) { @@ -1056,7 +1056,7 @@ async function setupMongoDb(app, options) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); - debug('Setting up mongodb'); + log('Setting up mongodb'); await ensureServiceRunning('mongodb'); @@ -1095,7 +1095,7 @@ async function setupMongoDb(app, options) { env.push({ name: 'CLOUDRON_MONGODB_OPLOG_URL', value : `mongodb://${data.username}:${data.password}@mongodb:27017/local?authSource=${data.database}` }); } - debug('Setting mongodb addon config to %j', env); + log('Setting mongodb addon config to %j', env); await addonConfigs.set(app.id, 'mongodb', env); } @@ -1141,7 +1141,7 @@ async function backupMongoDb(app, options) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); - debug('Backing up mongodb'); + log('Backing up mongodb'); if (!await hasAVX()) throw new BoxError(BoxError.ADDONS_ERROR, 'Error backing up MongoDB. CPU has no AVX support'); @@ -1159,7 +1159,7 @@ async function restoreMongoDb(app, options) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); - debug('restoreMongoDb'); + log('restoreMongoDb'); if (!await hasAVX()) throw new BoxError(BoxError.ADDONS_ERROR, 'Error restoring mongodb. CPU has no AVX support'); @@ -1193,7 +1193,7 @@ async function startGraphite(existingInfra) { const upgrading = existingInfra.version !== 'none' && requiresUpgrade(existingInfra.images.graphite, image); - if (upgrading) debug('startGraphite: graphite will be upgraded'); + if (upgrading) log('startGraphite: graphite will be upgraded'); const readOnly = !serviceConfig.recoveryMode ? '--read-only' : ''; const cmd = serviceConfig.recoveryMode ? '/bin/bash -c \'echo "Debug mode. Sleeping" && sleep infinity\'' : ''; @@ -1213,13 +1213,13 @@ async function startGraphite(existingInfra) { --label isCloudronManaged=true \ ${readOnly} -v /tmp -v /run ${image} ${cmd}`; - debug('startGraphite: stopping and deleting previous graphite container'); + log('startGraphite: stopping and deleting previous graphite container'); await docker.stopContainer('graphite'); await docker.deleteContainer('graphite'); if (upgrading) await shell.sudo([ RMADDONDIR_CMD, 'graphite' ], {}); - debug('startGraphite: starting graphite container'); + log('startGraphite: starting graphite container'); await shell.bash(runCmd, { encoding: 'utf8' }); if (existingInfra.version !== 'none' && existingInfra.images.graphite !== image) await docker.deleteImage(existingInfra.images.graphite); @@ -1229,7 +1229,7 @@ async function setupProxyAuth(app, options) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); - debug('Setting up proxyAuth'); + log('Setting up proxyAuth'); const enabled = app.sso && app.manifest.addons && app.manifest.addons.proxyAuth; @@ -1238,7 +1238,7 @@ async function setupProxyAuth(app, options) { const env = [ { name: 'CLOUDRON_PROXY_AUTH', value: '1' } ]; await addonConfigs.set(app.id, 'proxyauth', env); - debug('Creating OpenID client for proxyAuth'); + log('Creating OpenID client for proxyAuth'); const proxyAuthClientId = `${app.id}-proxyauth`; const result = await oidcClients.get(proxyAuthClientId); @@ -1263,7 +1263,7 @@ async function teardownProxyAuth(app, options) { await addonConfigs.unset(app.id, 'proxyauth'); - debug('Deleting OpenID client for proxyAuth'); + log('Deleting OpenID client for proxyAuth'); const proxyAuthClientId = `${app.id}-proxyauth`; @@ -1275,7 +1275,7 @@ async function setupDocker(app, options) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); - debug('Setting up docker'); + log('Setting up docker'); const env = [ { name: 'CLOUDRON_DOCKER_HOST', value: `tcp://172.18.0.1:${constants.DOCKER_PROXY_PORT}` } ]; await addonConfigs.set(app.id, 'docker', env); @@ -1343,7 +1343,7 @@ async function setupRedis(app, options) { if (inspectError) { await shell.bash(runCmd, { encoding: 'utf8' }); } else { // fast path - debug(`Re-using existing redis container with state: ${JSON.stringify(result.State)}`); + log(`Re-using existing redis container with state: ${JSON.stringify(result.State)}`); } if (isAppRunning && !recoveryMode) await waitForContainer(`redis-${app.id}`, 'CLOUDRON_REDIS_TOKEN'); @@ -1353,7 +1353,7 @@ async function clearRedis(app, options) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); - debug('Clearing redis'); + log('Clearing redis'); const disabled = app.manifest.addons.redis.optional && !app.enableRedis; if (disabled) return; @@ -1377,7 +1377,7 @@ async function teardownRedis(app, options) { if (error) throw new BoxError(BoxError.FS_ERROR, `Error removing redis data: ${error.message}`); safe.fs.rmSync(path.join(paths.LOG_DIR, `redis-${app.id}`), { recursive: true, force: true }); - if (safe.error) debug('teardownRedis: cannot cleanup logs: %o', safe.error); + if (safe.error) log('teardownRedis: cannot cleanup logs: %o', safe.error); await addonConfigs.unset(app.id, 'redis'); } @@ -1389,7 +1389,7 @@ async function backupRedis(app, options) { const disabled = app.manifest.addons.redis.optional && !app.enableRedis; if (disabled) return; - debug('Backing up redis'); + log('Backing up redis'); const result = await getContainerDetails('redis-' + app.id, 'CLOUDRON_REDIS_TOKEN'); const request = http.request(`http://${result.ip}:3000/backup?access_token=${result.token}`, { method: 'POST' }); @@ -1412,11 +1412,11 @@ async function startRedis(existingInfra) { if (upgrading) await backupRedis(app, {}); - debug(`startRedis: stopping and deleting previous redis container ${redisName}`); + log(`startRedis: stopping and deleting previous redis container ${redisName}`); await docker.stopContainer(redisName); await docker.deleteContainer(redisName); - debug(`startRedis: starting redis container ${redisName}`); + log(`startRedis: starting redis container ${redisName}`); await setupRedis(app, app.manifest.addons.redis); // starts the container } @@ -1432,7 +1432,7 @@ async function restoreRedis(app, options) { const disabled = app.manifest.addons.redis.optional && !app.enableRedis; if (disabled) return; - debug('Restoring redis'); + log('Restoring redis'); const result = await getContainerDetails('redis-' + app.id, 'CLOUDRON_REDIS_TOKEN'); const request = http.request(`http://${result.ip}:3000/restore?access_token=${result.token}`, { method: 'POST' }); @@ -1445,7 +1445,7 @@ async function setupTls(app, options) { assert.strictEqual(typeof options, 'object'); if (!safe.fs.mkdirSync(`${paths.PLATFORM_DATA_DIR}/tls/${app.id}`, { recursive: true })) { - debug('Error creating tls directory'); + log('Error creating tls directory'); throw new BoxError(BoxError.FS_ERROR, safe.error.message); } } @@ -1483,7 +1483,7 @@ async function statusDocker() { async function restartDocker() { const [error] = await safe(shell.sudo([ RESTART_SERVICE_CMD, 'docker' ], {})); - if (error) debug(`restartDocker: error restarting docker. ${error.message}`); + if (error) log(`restartDocker: error restarting docker. ${error.message}`); } async function statusUnbound() { @@ -1493,13 +1493,13 @@ async function statusUnbound() { const [digError, digResult] = await safe(dig.resolve('ipv4.api.cloudron.io', 'A', { timeout: 10000 })); if (!digError && Array.isArray(digResult) && digResult.length !== 0) return { status: SERVICE_STATUS_ACTIVE }; - debug('statusUnbound: unbound is up, but failed to resolve ipv4.api.cloudron.io . %o %j', digError, digResult); + log('statusUnbound: unbound is up, but failed to resolve ipv4.api.cloudron.io . %o %j', digError, digResult); return { status: SERVICE_STATUS_STARTING }; } async function restartUnbound() { const [error] = await safe(shell.sudo([ RESTART_SERVICE_CMD, 'unbound' ], {})); - if (error) debug(`restartDocker: error restarting unbound. ${error.message}`); + if (error) log(`restartDocker: error restarting unbound. ${error.message}`); } async function statusNginx() { @@ -1509,7 +1509,7 @@ async function statusNginx() { async function restartNginx() { const [error] = await safe(shell.sudo([ RESTART_SERVICE_CMD, 'nginx' ], {})); - if (error) debug(`restartNginx: error restarting unbound. ${error.message}`); + if (error) log(`restartNginx: error restarting unbound. ${error.message}`); } async function statusGraphite() { @@ -1676,7 +1676,7 @@ async function getServiceLogs(id, options) { throw new BoxError(BoxError.NOT_FOUND, 'Service not found'); } - debug(`getServiceLogs: getting logs for ${name}`); + log(`getServiceLogs: getting logs for ${name}`); let cp; @@ -1730,7 +1730,7 @@ async function applyMemoryLimit(id) { memoryLimit = serviceConfig && serviceConfig.memoryLimit ? serviceConfig.memoryLimit : APP_SERVICES[name].defaultMemoryLimit; } else if (SERVICES[name]) { if (name === 'mongodb' && !await hasAVX()) { - debug('applyMemoryLimit: skipping mongodb because CPU does not have AVX'); + log('applyMemoryLimit: skipping mongodb because CPU does not have AVX'); return; } @@ -1743,12 +1743,12 @@ async function applyMemoryLimit(id) { if (LAZY_SERVICES.includes(name)) { const [error, container] = await safe(docker.inspect(containerName)); if (error || !container.State?.Running) { - debug(`applyMemoryLimit: skipping ${containerName} (not running)`); + log(`applyMemoryLimit: skipping ${containerName} (not running)`); return; } } - debug(`applyMemoryLimit: ${containerName} ${JSON.stringify(serviceConfig)}`); + log(`applyMemoryLimit: ${containerName} ${JSON.stringify(serviceConfig)}`); await docker.update(containerName, memoryLimit); } @@ -1766,7 +1766,7 @@ async function applyServiceLimits() { changed = true; } - safe(applyMemoryLimit(id), { debug }); + safe(applyMemoryLimit(id), { debug: log }); } if (changed) await settings.setJson(settings.SERVICES_CONFIG_KEY, servicesConfig); @@ -1783,7 +1783,7 @@ async function applyServiceLimits() { await apps.update(app.id, { servicesConfig: app.servicesConfig }); } - safe(applyMemoryLimit(`redis:${app.id}`), { debug }); + safe(applyMemoryLimit(`redis:${app.id}`), { debug: log }); } } @@ -1797,7 +1797,7 @@ async function startTurn(existingInfra) { let turnSecret = await blobs.getString(blobs.ADDON_TURN_SECRET); if (!turnSecret) { - debug('startTurn: generating turn secret'); + log('startTurn: generating turn secret'); turnSecret = 'a' + crypto.randomBytes(15).toString('hex'); // prefix with a to ensure string starts with a letter await blobs.setString(blobs.ADDON_TURN_SECRET, turnSecret); } @@ -1825,11 +1825,11 @@ async function startTurn(existingInfra) { --label isCloudronManaged=true \ ${readOnly} -v /tmp -v /run ${image} ${cmd}`; - debug('startTurn: stopping and deleting previous turn container'); + log('startTurn: stopping and deleting previous turn container'); await docker.stopContainer('turn'); await docker.deleteContainer('turn'); - debug('startTurn: starting turn container'); + log('startTurn: starting turn container'); await shell.bash(runCmd, { encoding: 'utf8' }); if (existingInfra.version !== 'none' && existingInfra.images.turn !== image) await docker.deleteImage(existingInfra.images.turn); @@ -1877,7 +1877,7 @@ async function rebuildService(id, auditSource) { // nothing to rebuild for now. } - safe(applyMemoryLimit(id), { debug }); // do this in background. ok to fail + safe(applyMemoryLimit(id), { debug: log }); // do this in background. ok to fail await eventlog.add(eventlog.ACTION_SERVICE_REBUILD, auditSource, { id }); } @@ -1912,13 +1912,13 @@ async function configureService(id, data, auditSource) { throw new BoxError(BoxError.NOT_FOUND, 'No such service'); } - debug(`configureService: ${id} rebuild=${needsRebuild}`); + log(`configureService: ${id} rebuild=${needsRebuild}`); // do this in background if (needsRebuild) { - safe(rebuildService(id, auditSource), { debug }); + safe(rebuildService(id, auditSource), { debug: log }); } else { - safe(applyMemoryLimit(id), { debug }); + safe(applyMemoryLimit(id), { debug: log }); } await eventlog.add(eventlog.ACTION_SERVICE_CONFIGURE, auditSource, { id, data }); @@ -1950,14 +1950,14 @@ async function startServices(existingInfra, progressCallback) { await fn(existingInfra); } - safe(applyServiceLimits(), { debug }); + safe(applyServiceLimits(), { debug: log }); } async function teardownOauth(app, options) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); - debug('teardownOauth'); + log('teardownOauth'); await addonConfigs.unset(app.id, 'oauth'); } @@ -1968,7 +1968,7 @@ async function setupOidc(app, options) { if (!app.sso) return; - debug('Setting up OIDC'); + log('Setting up OIDC'); const oidcAddonClientId = `${app.id}-oidc`; const [error, result] = await safe(oidcClients.get(oidcAddonClientId)); @@ -1993,7 +1993,7 @@ async function teardownOidc(app, options) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); - debug('Tearing down OIDC'); + log('Tearing down OIDC'); const oidcAddonClientId = `${app.id}-oidc`; @@ -2047,7 +2047,7 @@ async function moveDataDir(app, targetVolumeId, targetVolumePrefix) { const resolvedSourceDir = await apps.getStorageDir(app); const resolvedTargetDir = await apps.getStorageDir(Object.assign({}, app, { storageVolumeId: targetVolumeId, storageVolumePrefix: targetVolumePrefix })); - debug(`moveDataDir: migrating data from ${resolvedSourceDir} to ${resolvedTargetDir}`); + log(`moveDataDir: migrating data from ${resolvedSourceDir} to ${resolvedTargetDir}`); if (resolvedSourceDir !== resolvedTargetDir) { const [error] = await safe(shell.sudo([ MV_VOLUME_CMD, resolvedSourceDir, resolvedTargetDir ], {})); @@ -2203,12 +2203,12 @@ async function setupAddons(app, addons) { if (!addons) return; - debug('setupAddons: Setting up %j', Object.keys(addons)); + log('setupAddons: Setting up %j', Object.keys(addons)); for (const addon of Object.keys(addons)) { if (!(addon in ADDONS)) throw new BoxError(BoxError.NOT_FOUND, `No such addon: ${addon}`); - debug(`setupAddons: setting up addon ${addon} with options ${JSON.stringify(addons[addon])}`); + log(`setupAddons: setting up addon ${addon} with options ${JSON.stringify(addons[addon])}`); await ADDONS[addon].setup(app, addons[addon]); } @@ -2220,12 +2220,12 @@ async function teardownAddons(app, addons) { if (!addons) return; - debug('teardownAddons: Tearing down %j', Object.keys(addons)); + log('teardownAddons: Tearing down %j', Object.keys(addons)); for (const addon of Object.keys(addons)) { if (!(addon in ADDONS)) throw new BoxError(BoxError.NOT_FOUND, `No such addon: ${addon}`); - debug(`teardownAddons: Tearing down addon ${addon} with options ${JSON.stringify(addons[addon])}`); + log(`teardownAddons: Tearing down addon ${addon} with options ${JSON.stringify(addons[addon])}`); await ADDONS[addon].teardown(app, addons[addon]); } @@ -2237,7 +2237,7 @@ async function backupAddons(app, addons) { if (!addons) return; - debug('backupAddons: backing up %j', Object.keys(addons)); + log('backupAddons: backing up %j', Object.keys(addons)); for (const addon of Object.keys(addons)) { if (!(addon in ADDONS)) throw new BoxError(BoxError.NOT_FOUND, `No such addon: ${addon}`); @@ -2252,7 +2252,7 @@ async function clearAddons(app, addons) { if (!addons) return; - debug('clearAddons: clearing %j', Object.keys(addons)); + log('clearAddons: clearing %j', Object.keys(addons)); for (const addon of Object.keys(addons)) { if (!(addon in ADDONS)) throw new BoxError(BoxError.NOT_FOUND, `No such addon: ${addon}`); @@ -2268,7 +2268,7 @@ async function restoreAddons(app, addons) { if (!addons) return; - debug('restoreAddons: restoring %j', Object.keys(addons)); + log('restoreAddons: restoring %j', Object.keys(addons)); for (const addon of Object.keys(addons)) { if (!(addon in ADDONS)) throw new BoxError(BoxError.NOT_FOUND, `No such addon: ${addon}`); diff --git a/src/sftp.js b/src/sftp.js index 83806fee7..a23940e69 100644 --- a/src/sftp.js +++ b/src/sftp.js @@ -3,7 +3,7 @@ import assert from 'node:assert'; import blobs from './blobs.js'; import BoxError from './boxerror.js'; import constants from './constants.js'; -import debugModule from 'debug'; +import logger from './logger.js'; import docker from './docker.js'; import hat from './hat.js'; import infra from './infra_version.js'; @@ -15,7 +15,7 @@ import services from './services.js'; import shellModule from './shell.js'; import volumes from './volumes.js'; -const debug = debugModule('box:sftp'); +const { log, trace } = logger('sftp'); const shell = shellModule('sftp'); const DEFAULT_MEMORY_LIMIT = 256 * 1024 * 1024; @@ -29,7 +29,7 @@ async function ensureKeys() { const privateKeyFile = path.join(paths.SFTP_KEYS_DIR, `ssh_host_${keyType}_key`); if (!privateKey || !publicKey) { - debug(`ensureSecrets: generating new sftp keys of type ${keyType}`); + log(`ensureSecrets: generating new sftp keys of type ${keyType}`); safe.fs.unlinkSync(publicKeyFile); safe.fs.unlinkSync(privateKeyFile); const [error] = await safe(shell.spawn('ssh-keygen', ['-m', 'PEM', '-t', keyType, '-f', `${paths.SFTP_KEYS_DIR}/ssh_host_${keyType}_key`, '-q', '-N', ''], {})); @@ -48,7 +48,7 @@ async function ensureKeys() { async function start(existingInfra) { assert.strictEqual(typeof existingInfra, 'object'); - debug('start: re-creating container'); + log('start: re-creating container'); const serviceConfig = await services.getServiceConfig('sftp'); const image = infra.images.sftp; @@ -70,7 +70,7 @@ async function start(existingInfra) { const hostDir = await apps.getStorageDir(app), mountDir = `/mnt/app-${app.id}`; // see also sftp:userSearchSftp if (hostDir === null || !safe.fs.existsSync(hostDir)) { // this can fail if external mount does not have permissions for yellowtent user // do not create host path when cloudron is restoring. this will then create dir with root perms making restore logic fail - debug(`Ignoring app data dir ${hostDir} for ${app.id} since it does not exist`); + log(`Ignoring app data dir ${hostDir} for ${app.id} since it does not exist`); continue; } @@ -84,7 +84,7 @@ async function start(existingInfra) { if (mounts.isManagedProvider(volume.mountType)) continue; // skip managed volume. these are acessed via /mnt/volumes mount above if (!safe.fs.existsSync(volume.hostPath)) { - debug(`Ignoring volume host path ${volume.hostPath} since it does not exist`); + log(`Ignoring volume host path ${volume.hostPath} since it does not exist`); continue; } @@ -118,11 +118,11 @@ async function start(existingInfra) { --label isCloudronManaged=true \ ${readOnly} -v /tmp -v /run ${image} ${cmd}`; - debug('startSftp: stopping and deleting previous sftp container'); + log('startSftp: stopping and deleting previous sftp container'); await docker.stopContainer('sftp'); await docker.deleteContainer('sftp'); - debug('startSftp: starting sftp container'); + log('startSftp: starting sftp container'); await shell.bash(runCmd, { encoding: 'utf8' }); if (existingInfra.version !== 'none' && existingInfra.images.sftp !== image) await docker.deleteImage(existingInfra.images.sftp); diff --git a/src/shell.js b/src/shell.js index 4e5b45d4f..2ae8805fe 100644 --- a/src/shell.js +++ b/src/shell.js @@ -1,12 +1,12 @@ import assert from 'node:assert'; import BoxError from './boxerror.js'; import child_process from 'node:child_process'; -import debugModule from 'debug'; +import logger from './logger.js'; import path from 'node:path'; import safe from 'safetydance'; import _ from './underscore.js'; -const debug = debugModule('box:shell'); +const { log, trace } = logger('shell'); function lineCount(buffer) { assert(Buffer.isBuffer(buffer)); @@ -30,7 +30,7 @@ function spawn(tag, file, args, options) { assert(Array.isArray(args)); assert.strictEqual(typeof options, 'object'); // note: spawn() has no encoding option of it's own - debug(`${tag}: ${file} ${args.join(' ').replace(/\n/g, '\\n')}`); + log(`${tag}: ${file} ${args.join(' ').replace(/\n/g, '\\n')}`); const maxLines = options.maxLines || Number.MAX_SAFE_INTEGER; const logger = options.logger || null; @@ -79,26 +79,26 @@ function spawn(tag, file, args, options) { e.timedOut = timedOut; e.terminated = terminated; - debug(`${tag}: ${file} ${args.join(' ').replace(/\n/g, '\\n')} errored`, e); + log(`${tag}: ${file} ${args.join(' ').replace(/\n/g, '\\n')} errored`, e); reject(e); }); cp.on('error', function (error) { // when the command itself could not be started - debug(`${tag}: ${file} ${args.join(' ').replace(/\n/g, '\\n')} errored`, error); + log(`${tag}: ${file} ${args.join(' ').replace(/\n/g, '\\n')} errored`, error); }); cp.terminate = function () { terminated = true; // many approaches to kill sudo launched process failed. we now have a sudo wrapper to kill the full tree child_process.execFile('/usr/bin/sudo', [ KILL_CHILD_CMD, cp.pid, process.pid ], { encoding: 'utf8' }, (error, stdout, stderr) => { - if (error) debug(`${tag}: failed to kill children`, stdout, stderr); - else debug(`${tag}: terminated ${cp.pid}`, stdout, stderr); + if (error) log(`${tag}: failed to kill children`, stdout, stderr); + else log(`${tag}: terminated ${cp.pid}`, stdout, stderr); }); }; abortSignal?.addEventListener('abort', () => { - debug(`${tag}: aborting ${cp.pid}`); + log(`${tag}: aborting ${cp.pid}`); cp.terminate(); }, { once: true }); @@ -106,10 +106,10 @@ function spawn(tag, file, args, options) { if (options.timeout) { killTimerId = setTimeout(async () => { - debug(`${tag}: timedout`); + log(`${tag}: timedout`); timedOut = true; if (typeof options.onTimeout !== 'function') return cp.terminate(); - await safe(options.onTimeout(), { debug }); + await safe(options.onTimeout(), { debug: log }); }, options.timeout); } diff --git a/src/storage/filesystem.js b/src/storage/filesystem.js index 85b0ae6d0..e19b5ce6b 100644 --- a/src/storage/filesystem.js +++ b/src/storage/filesystem.js @@ -1,7 +1,7 @@ import assert from 'node:assert'; import BoxError from '../boxerror.js'; import crypto from 'crypto'; -import debugModule from 'debug'; +import logger from '../logger.js'; import df from '../df.js'; import fs from 'node:fs'; import mounts from '../mounts.js'; @@ -11,7 +11,7 @@ import safe from 'safetydance'; import shellModule from '../shell.js'; import _ from '../underscore.js'; -const debug = debugModule('box:storage/filesystem'); +const { log, trace } = logger('storage/filesystem'); const shell = shellModule('filesystem'); @@ -111,7 +111,7 @@ async function download(config, remotePath) { assert.strictEqual(typeof remotePath, 'string'); const fullRemotePath = path.join(getRootPath(config), remotePath); - debug(`download: ${fullRemotePath}`); + log(`download: ${fullRemotePath}`); if (!safe.fs.existsSync(fullRemotePath)) throw new BoxError(BoxError.NOT_FOUND, `File not found: ${fullRemotePath}`); @@ -208,7 +208,7 @@ async function copyInternal(config, fromPath, toPath, options, progressCallback) safe.fs.unlinkSync(identityFilePath); if (!remoteCopyError) return; if (remoteCopyError.code === 255) throw new BoxError(BoxError.EXTERNAL_ERROR, `SSH connection error: ${remoteCopyError.message}`); // do not attempt fallback copy for ssh errors - debug('SSH remote copy failed, trying sshfs copy'); // this can happen for sshfs mounted windows server + log('SSH remote copy failed, trying sshfs copy'); // this can happen for sshfs mounted windows server } const [copyError] = await safe(shell.spawn('cp', [ cpOptions, fullFromPath, fullToPath ], {})); @@ -271,7 +271,7 @@ async function removeDir(config, limits, remotePathPrefix, progressCallback) { safe.fs.unlinkSync(identityFilePath); if (!remoteRmError) return; if (remoteRmError.code === 255) throw new BoxError(BoxError.EXTERNAL_ERROR, `SSH connection error: ${remoteRmError.message}`); // do not attempt fallback copy for ssh errors - debug('SSH remote rm failed, trying sshfs rm'); // this can happen for sshfs mounted windows server + log('SSH remote rm failed, trying sshfs rm'); // this can happen for sshfs mounted windows server } const [error] = await safe(shell.spawn('rm', [ '-rf', fullPathPrefix ], {})); @@ -311,13 +311,13 @@ function mountObjectFromConfig(config) { async function setup(config) { assert.strictEqual(typeof config, 'object'); - debug('setup: removing old storage configuration'); + log('setup: removing old storage configuration'); if (!mounts.isManagedProvider(config._provider)) return; const mount = mountObjectFromConfig(config); - await safe(mounts.removeMount(mount), { debug }); // ignore error + await safe(mounts.removeMount(mount), { debug: log }); // ignore error - debug('setup: setting up new storage configuration'); + log('setup: setting up new storage configuration'); await mounts.tryAddMount(mount, { timeout: 10 }); // 10 seconds } @@ -326,7 +326,7 @@ async function teardown(config) { if (!mounts.isManagedProvider(config._provider)) return; - await safe(mounts.removeMount(mountObjectFromConfig(config)), { debug }); // ignore error + await safe(mounts.removeMount(mountObjectFromConfig(config)), { debug: log }); // ignore error } async function verifyConfig({ id, provider, config }) { diff --git a/src/storage/gcs.js b/src/storage/gcs.js index 4dfaba327..55b4578b2 100644 --- a/src/storage/gcs.js +++ b/src/storage/gcs.js @@ -2,13 +2,13 @@ import assert from 'node:assert'; import async from 'async'; import BoxError from '../boxerror.js'; import constants from '../constants.js'; -import debugModule from 'debug'; +import logger from '../logger.js'; import { Storage as GCS } from '@google-cloud/storage'; import path from 'node:path'; import safe from 'safetydance'; import _ from '../underscore.js'; -const debug = debugModule('box:storage/gcs'); +const { log, trace } = logger('storage/gcs'); function getBucket(config) { @@ -50,7 +50,7 @@ async function upload(config, limits, remotePath) { const fullRemotePath = path.join(config.prefix, remotePath); - debug(`Uploading to ${fullRemotePath}`); + log(`Uploading to ${fullRemotePath}`); return { createStream() { return getBucket(config).file(fullRemotePath).createWriteStream({ resumable: false }); }, @@ -91,7 +91,7 @@ async function download(config, remotePath) { assert.strictEqual(typeof remotePath, 'string'); const fullRemotePath = path.join(config.prefix, remotePath); - debug(`Download ${fullRemotePath} starting`); + log(`Download ${fullRemotePath} starting`); const file = getBucket(config).file(fullRemotePath); return file.createReadStream(); @@ -124,7 +124,7 @@ async function copyInternal(config, fullFromPath, fullToPath, progressCallback) assert.strictEqual(typeof progressCallback, 'function'); const [copyError] = await safe(getBucket(config).file(fullFromPath).copy(fullToPath)); - if (copyError) debug('copyBackup: gcs copy error. %o', copyError); + if (copyError) log('copyBackup: gcs copy error. %o', copyError); if (copyError && copyError.code === 404) throw new BoxError(BoxError.NOT_FOUND, 'Old backup not found'); if (copyError) throw new BoxError(BoxError.EXTERNAL_ERROR, copyError.message); } @@ -175,7 +175,7 @@ async function remove(config, remotePath) { const fullRemotePath = path.join(config.prefix, remotePath); const [error] = await safe(getBucket(config).file(fullRemotePath).delete()); - if (error) debug('removeBackups: Unable to remove %s (%s). Not fatal.', fullRemotePath, error.message); + if (error) log('removeBackups: Unable to remove %s (%s). Not fatal.', fullRemotePath, error.message); } async function removeDir(config, limits, remotePathPrefix, progressCallback) { @@ -226,11 +226,11 @@ async function verifyConfig({ id, provider, config }) { const testFile = bucket.file(path.join(config.prefix, 'snapshot/cloudron-testfile')); const [saveError] = await safe(testFile.save('testfilecontents', { resumable: false })); if (saveError) { // with bad creds, it return status 500 and "Cannot call write after a stream was destroyed" - debug('testConfig: failed uploading cloudron-testfile. %o', saveError); + log('testConfig: failed uploading cloudron-testfile. %o', saveError); if (saveError.code == 403 || saveError.code == 404) throw new BoxError(BoxError.BAD_FIELD, saveError.message); throw new BoxError(BoxError.EXTERNAL_ERROR, saveError.message); } - debug('testConfig: uploaded cloudron-testfile'); + log('testConfig: uploaded cloudron-testfile'); const query = { prefix: path.join(config.prefix, 'snapshot'), autoPaginate: false, maxResults: 1 }; const [listError] = await safe(bucket.getFiles(query)); @@ -238,7 +238,7 @@ async function verifyConfig({ id, provider, config }) { const [delError] = await safe(bucket.file(path.join(config.prefix, 'snapshot/cloudron-testfile')).delete()); if (delError) throw new BoxError(BoxError.EXTERNAL_ERROR, delError.message); - debug('testConfig: deleted cloudron-testfile'); + log('testConfig: deleted cloudron-testfile'); return _.pick(config, ['projectId', 'credentials', 'bucket', 'prefix']); } diff --git a/src/storage/s3.js b/src/storage/s3.js index 64a2a1877..b5b397e83 100644 --- a/src/storage/s3.js +++ b/src/storage/s3.js @@ -5,7 +5,7 @@ import { ConfiguredRetryStrategy } from '@smithy/util-retry'; import constants from '../constants.js'; import consumers from 'node:stream/consumers'; import crypto from 'node:crypto'; -import debugModule from 'debug'; +import logger from '../logger.js'; import http from 'node:http'; import https from 'node:https'; import { NodeHttpHandler } from '@smithy/node-http-handler'; @@ -17,7 +17,7 @@ import safe from 'safetydance'; import { Upload } from '@aws-sdk/lib-storage'; import _ from '../underscore.js'; -const debug = debugModule('box:storage/s3'); +const { log, trace } = logger('storage/s3'); function S3_NOT_FOUND(error) { return error instanceof NoSuchKey || error instanceof NoSuchBucket; @@ -93,8 +93,8 @@ function createS3Client(config, options) { const client = constants.TEST ? new globalThis.S3Mock(clientConfig) : new S3(clientConfig); // https://github.com/aws/aws-sdk-js-v3/issues/6761#issuecomment-2574480834 // client.middlewareStack.add((next, context) => async (args) => { - // debug('AWS SDK context', context.clientName, context.commandName); - // debug('AWS SDK request input', JSON.stringify(args.input)); + // log('AWS SDK context', context.clientName, context.commandName); + // log('AWS SDK request input', JSON.stringify(args.input)); // const result = await next(args); // console.log('AWS SDK request output:', result.output); // return result; @@ -174,7 +174,7 @@ async function upload(config, limits, remotePath) { }; const managedUpload = constants.TEST ? new globalThis.S3MockUpload(options) : new Upload(options); - // managedUpload.on('httpUploadProgress', (progress) => debug(`Upload progress: ${JSON.stringify(progress)}`)); + // managedUpload.on('httpUploadProgress', (progress) => log(`Upload progress: ${JSON.stringify(progress)}`)); uploadPromise = managedUpload.done(); return passThrough; @@ -183,7 +183,7 @@ async function upload(config, limits, remotePath) { if (!uploadPromise) return; // stream was never created const [error/*,data*/] = await safe(uploadPromise); if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, `Upload error: code: ${error.Code} message: ${error.message}`); // sometimes message is null - // debug(`Upload finished. ${JSON.stringify(data)}`); // { ETag, $metadata:{httpStatusCode,requestId,attempts,totalRetryDelay},Bucket,Key} + // log(`Upload finished. ${JSON.stringify(data)}`); // { ETag, $metadata:{httpStatusCode,requestId,attempts,totalRetryDelay},Bucket,Key} } }; } @@ -245,7 +245,7 @@ class S3MultipartDownloadStream extends Readable { if (S3_NOT_FOUND(error)) { this.destroy(new BoxError(BoxError.NOT_FOUND, `Backup not found: ${this._path}`)); } else { - debug(`download: ${this._path} s3 stream error. %o`, error); + log(`download: ${this._path} s3 stream error. %o`, error); this.destroy(new BoxError(BoxError.EXTERNAL_ERROR, `Error multipartDownload ${this._path}. ${formatError(error)}`)); } } @@ -443,7 +443,7 @@ async function cleanup(config, progressCallback) { for (const multipartUpload of uploads.Uploads) { if (Date.now() - new Date(multipartUpload.Initiated) < 3 * 24 * 60 * 60 * 1000) continue; // 3 days ago progressCallback({ message: `Cleaning up multi-part upload uploadId:${multipartUpload.UploadId} key:${multipartUpload.Key}` }); - await safe(s3.abortMultipartUpload({ Bucket: config.bucket, Key: multipartUpload.Key, UploadId: multipartUpload.UploadId }), { debug }); // ignore error + await safe(s3.abortMultipartUpload({ Bucket: config.bucket, Key: multipartUpload.Key, UploadId: multipartUpload.UploadId }), { debug: log }); // ignore error } } @@ -535,7 +535,7 @@ async function copyInternal(config, fullFromPath, fullToPath, fileSize, progress const s3 = createS3Client(config, { retryStrategy: RETRY_STRATEGY }); // https://docs.aws.amazon.com/sdkref/latest/guide/feature-retry-behavior.html function throwError(error) { - if (error) debug(`copy: s3 copy error when copying ${fullFromPath}: ${error}`); + if (error) log(`copy: s3 copy error when copying ${fullFromPath}: ${error}`); if (error && S3_NOT_FOUND(error)) throw new BoxError(BoxError.NOT_FOUND, `Old backup not found: ${fullFromPath}`); if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, `Error copying ${fullFromPath} (${fileSize} bytes): ${error.Code || ''} ${error}`); @@ -603,7 +603,7 @@ async function copyInternal(config, fullFromPath, fullToPath, fileSize, progress UploadId: uploadId }; progressCallback({ message: `Aborting multipart copy of ${fullFromPath}` }); - await safe(s3.abortMultipartUpload(abortParams), { debug }); // ignore any abort errors + await safe(s3.abortMultipartUpload(abortParams), { debug: log }); // ignore any abort errors return throwError(copyError); } diff --git a/src/syncer.js b/src/syncer.js index d63390a93..c1a6d8291 100644 --- a/src/syncer.js +++ b/src/syncer.js @@ -1,14 +1,14 @@ import assert from 'node:assert'; import BoxError from './boxerror.js'; import DataLayout from './datalayout.js'; -import debugModule from 'debug'; +import logger from './logger.js'; import fs from 'node:fs'; import path from 'node:path'; import readline from 'node:readline'; import safe from 'safetydance'; import util from 'node:util'; -const debug = debugModule('box:syncer'); +const { log, trace } = logger('syncer'); function readCache(cacheFile) { @@ -72,13 +72,13 @@ async function sync(dataLayout, cacheFile) { // if cache is missing or if we crashed/errored in previous run, start out empty if (!safe.fs.existsSync(cacheFile)) { - debug(`sync: cache file ${cacheFile} is missing, starting afresh`); + log(`sync: cache file ${cacheFile} is missing, starting afresh`); delQueue.push({ operation: 'removedir', path: '', reason: 'nocache' }); } else if (safe.fs.existsSync(newCacheFile)) { - debug(`sync: new cache file ${newCacheFile} exists. previous run crashed, starting afresh`); + log(`sync: new cache file ${newCacheFile} exists. previous run crashed, starting afresh`); delQueue.push({ operation: 'removedir', path: '', reason: 'crash' }); } else { - debug(`sync: loading cache file ${cacheFile}`); + log(`sync: loading cache file ${cacheFile}`); cache = readCache(cacheFile); } @@ -168,7 +168,7 @@ async function finalize(integrityMap, cacheFile) { const newCacheFile = `${cacheFile}.new`, tempCacheFile = `${cacheFile}.tmp`; - debug(`finalize: patching in integrity information into ${cacheFile}`); + log(`finalize: patching in integrity information into ${cacheFile}`); const tempCacheFd = safe.fs.openSync(tempCacheFile, 'w'); // truncates any existing file if (tempCacheFd === -1) throw new BoxError(BoxError.FS_ERROR, 'Error opening temp cache file: ' + safe.error.message); @@ -192,7 +192,7 @@ async function finalize(integrityMap, cacheFile) { safe.fs.unlinkSync(cacheFile); safe.fs.unlinkSync(newCacheFile); - if (!safe.fs.renameSync(tempCacheFile, cacheFile)) debug('Unable to save new cache file'); + if (!safe.fs.renameSync(tempCacheFile, cacheFile)) log('Unable to save new cache file'); } export default { diff --git a/src/system.js b/src/system.js index 46c8dbaeb..0fabc326f 100644 --- a/src/system.js +++ b/src/system.js @@ -4,7 +4,7 @@ import asynctask from './asynctask.js'; const { AsyncTask } = asynctask; import backupSites from './backupsites.js'; import BoxError from './boxerror.js'; -import debugModule from 'debug'; +import logger from './logger.js'; import df from './df.js'; import docker from './docker.js'; import eventlog from './eventlog.js'; @@ -18,7 +18,7 @@ import safe from 'safetydance'; import shellModule from './shell.js'; import volumes from './volumes.js'; -const debug = debugModule('box:system'); +const { log, trace } = logger('system'); const shell = shellModule('system'); @@ -168,7 +168,7 @@ async function getFilesystems() { } async function checkDiskSpace() { - debug('checkDiskSpace: checking disk space'); + log('checkDiskSpace: checking disk space'); const filesystems = await getFilesystems(); @@ -185,7 +185,7 @@ async function checkDiskSpace() { } } - debug(`checkDiskSpace: disk space checked. low disk space: ${markdownMessage || 'no'}`); + log(`checkDiskSpace: disk space checked. low disk space: ${markdownMessage || 'no'}`); if (markdownMessage) { const finalMessage = `One or more file systems are running low on space. Please increase the disk size at the earliest.\n\n${markdownMessage}`; @@ -223,7 +223,7 @@ class FilesystemUsageTask extends AsyncTask { if (type === 'ext4' || type === 'xfs') { // hdparm only works with block devices this.emitProgress(percent, 'Calculating Disk Speed'); const [speedError, speed] = await safe(hdparm(filesystem, { abortSignal })); - if (speedError) debug(`hdparm error ${filesystem}: ${speedError.message}`); + if (speedError) log(`hdparm error ${filesystem}: ${speedError.message}`); this.emitData({ speed: speedError ? -1 : speed }); } else { this.emitData({ speed: -1 }); @@ -241,7 +241,7 @@ class FilesystemUsageTask extends AsyncTask { content.usage = content.id === 'docker' ? dockerDf.LayersSize : dockerDf.Volumes.map((v) => v.UsageData.Size).reduce((a,b) => a + b, 0); } else { const [error, duResult] = await safe(du(content.path, { abortSignal })); - if (error) debug(`du error ${content.path}: ${error.message}`); // can happen if app is installing etc + if (error) log(`du error ${content.path}: ${error.message}`); // can happen if app is installing etc content.usage = duResult || 0; } usage += content.usage; @@ -266,7 +266,7 @@ async function reboot() { await notifications.unpin(notifications.TYPE_REBOOT, {}); const [error] = await safe(shell.sudo([ REBOOT_CMD ], {})); - if (error) debug('reboot: could not reboot. %o', error); + if (error) log('reboot: could not reboot. %o', error); } async function getInfo() { @@ -369,7 +369,7 @@ async function checkUbuntuVersion() { } async function runSystemChecks() { - debug('runSystemChecks: checking status'); + log('runSystemChecks: checking status'); const checks = [ checkRebootRequired(), diff --git a/src/tasks.js b/src/tasks.js index a0e702792..e84aee07d 100644 --- a/src/tasks.js +++ b/src/tasks.js @@ -1,7 +1,7 @@ import assert from 'node:assert'; import BoxError from './boxerror.js'; import database from './database.js'; -import debugModule from 'debug'; +import logger from './logger.js'; import logs from './logs.js'; import mysql from 'mysql2'; import path from 'node:path'; @@ -10,7 +10,7 @@ import safe from 'safetydance'; import shellModule from './shell.js'; import _ from './underscore.js'; -const debug = debugModule('box:tasks'); +const { log, trace } = logger('tasks'); const shell = shellModule('tasks'); const ESTOPPED = 'stopped'; @@ -70,7 +70,7 @@ async function update(id, task) { assert.strictEqual(typeof id, 'string'); assert.strictEqual(typeof task, 'object'); - debug(`updating task ${id} with: ${JSON.stringify(task)}`); + log(`updating task ${id} with: ${JSON.stringify(task)}`); const args = [], fields = []; for (const k in task) { @@ -92,7 +92,7 @@ async function setCompleted(id, task) { assert.strictEqual(typeof id, 'string'); assert.strictEqual(typeof task, 'object'); - debug(`setCompleted - ${id}: ${JSON.stringify(task)}`); + log(`setCompleted - ${id}: ${JSON.stringify(task)}`); await update(id, Object.assign({ completed: true }, task)); } @@ -145,7 +145,7 @@ async function stopTask(id) { if (!gTasks[id]) throw new BoxError(BoxError.BAD_STATE, 'task is not active'); - debug(`stopTask: stopping task ${id}`); + log(`stopTask: stopping task ${id}`); await shell.sudo([ STOP_TASK_CMD, id, ], {}); // note: this is stopping the systemd-run task. the sudo will exit when this exits } @@ -155,7 +155,7 @@ async function startTask(id, options) { assert.strictEqual(typeof options, 'object'); const logFile = options.logFile || `${paths.TASKS_LOG_DIR}/${id}.log`; - debug(`startTask - starting task ${id} with options ${JSON.stringify(options)}. logs at ${logFile}`); + log(`startTask - starting task ${id} with options ${JSON.stringify(options)}. logs at ${logFile}`); const ac = new AbortController(); gTasks[id] = ac; @@ -166,17 +166,17 @@ async function startTask(id, options) { abortSignal: ac.signal, timeout: options.timeout || 0, onTimeout: async () => { // custom stop because kill won't do. the task is running in some other process tree - debug(`onTimeout: ${id}`); + log(`onTimeout: ${id}`); await stopTask(id); } }; - safe(update(id, { pending: false }), { debug }); // background. we have to create the cp immediately to prevent race with stopTask() + safe(update(id, { pending: false }), { debug: log }); // background. we have to create the cp immediately to prevent race with stopTask() const [sudoError] = await safe(shell.sudo([ START_TASK_CMD, id, logFile, options.nice || 0, options.memoryLimit || 400, options.oomScoreAdjust || 0 ], sudoOptions)); if (!gTasks[id]) { // when box code is shutting down, don't update the task status as "crashed". see stopAllTasks() - debug(`startTask: ${id} completed as a result of box shutdown`); + log(`startTask: ${id} completed as a result of box shutdown`); return null; } @@ -186,7 +186,7 @@ async function startTask(id, options) { if (!task) return null; // task disappeared on us. this can happen when db got cleared in tests if (task.completed) { // task completed. we can trust the db result - debug(`startTask: ${id} completed. error: %o`, task.error); + log(`startTask: ${id} completed. error: %o`, task.error); if (task.error) throw task.error; return task.result; } @@ -201,19 +201,19 @@ async function startTask(id, options) { else if (sudoError.code === 50) taskError = { message:`Task ${id} crashed with code ${sudoError.code}`, code: ECRASHED }; else taskError = { message:`Task ${id} crashed with unknown code ${sudoError.code}`, code: ECRASHED }; - debug(`startTask: ${id} done. error: %o`, taskError); - await safe(setCompleted(id, { error: taskError }), { debug }); + log(`startTask: ${id} done. error: %o`, taskError); + await safe(setCompleted(id, { error: taskError }), { debug: log }); throw taskError; } async function stopAllTasks() { const acs = Object.values(gTasks); - debug(`stopAllTasks: ${acs.length} tasks are running. sending abort signal`); + log(`stopAllTasks: ${acs.length} tasks are running. sending abort signal`); gTasks = {}; // this signals startTask() to not set completion status as "crashed" acs.forEach(ac => ac.abort()); // cleanup all the sudos and systemd-run const [error] = await safe(shell.sudo([ STOP_TASK_CMD, 'all' ], { cwd: paths.baseDir() })); - if (error) debug(`stopAllTasks: error stopping stasks: ${error.message}`); + if (error) log(`stopAllTasks: error stopping stasks: ${error.message}`); } async function getLogs(task, options) { diff --git a/src/taskworker.js b/src/taskworker.js index 9905abdd2..b2ed3b879 100755 --- a/src/taskworker.js +++ b/src/taskworker.js @@ -19,7 +19,7 @@ import safe from 'safetydance'; import tasks from './tasks.js'; import timers from 'timers/promises'; import updater from './updater.js'; -import debugModule from 'debug'; +import logger from './logger.js'; const TASKS = { // indexed by task type app: apptask.run, @@ -100,28 +100,28 @@ async function main() { return process.exit(50); } - const debug = debugModule('box:taskworker'); // import this here so that logging handler is already setup + const { log, trace } = logger('taskworker'); // import this here so that logging handler is already setup process.on('SIGTERM', () => { - debug('Terminated'); + log('Terminated'); exitSync({ code: 70 }); }); // ensure we log task crashes with the task logs. neither console.log nor debug are sync for some reason process.on('uncaughtException', (error) => exitSync({ error, code: 1 })); - debug(`Starting task ${taskId}. Logs are at ${logFile}`); + log(`Starting task ${taskId}. Logs are at ${logFile}`); const [getError, task] = await safe(tasks.get(taskId)); if (getError) return exitSync({ error: getError, code: 50 }); if (!task) return exitSync({ error: new Error(`Task ${taskId} not found`), code: 50 }); async function progressCallback(progress) { - await safe(tasks.update(taskId, progress), { debug }); + await safe(tasks.update(taskId, progress), { debug: log }); } const taskName = task.type.replace(/_.*/,''); - debug(`Running task of type ${taskName}`); + log(`Running task of type ${taskName}`); const [runError, result] = await safe(TASKS[taskName].apply(null, task.args.concat(progressCallback))); const progress = { result: result || null, @@ -129,9 +129,9 @@ async function main() { percent: 100 }; - await safe(tasks.setCompleted(taskId, progress), { debug }); + await safe(tasks.setCompleted(taskId, progress), { debug: log }); - debug(`Task took ${(new Date() - startTime)/1000} seconds`); + log(`Task took ${(new Date() - startTime)/1000} seconds`); exitSync({ error: runError, code: (!runError || runError instanceof BoxError) ? 0 : 50 }); // handled error vs run time crash } diff --git a/src/translations.js b/src/translations.js index db4c0ed70..0fce8d4e5 100644 --- a/src/translations.js +++ b/src/translations.js @@ -1,12 +1,12 @@ import assert from 'node:assert'; import cloudron from './cloudron.js'; -import debugModule from 'debug'; +import logger from './logger.js'; import fs from 'node:fs'; import path from 'node:path'; import paths from './paths.js'; import safe from 'safetydance'; -const debug = debugModule('box:translation'); +const { log, trace } = logger('translation'); // to be used together with getTranslations() => { translations, fallback } @@ -45,13 +45,13 @@ function translate(input, assets) { async function getTranslations() { const fallbackData = fs.readFileSync(path.join(paths.TRANSLATIONS_DIR, 'en.json'), 'utf8'); - if (!fallbackData) debug(`getTranslations: Fallback language en not found. ${safe.error.message}`); + if (!fallbackData) log(`getTranslations: Fallback language en not found. ${safe.error.message}`); const fallback = safe.JSON.parse(fallbackData) || {}; const lang = await cloudron.getLanguage(); const translationData = safe.fs.readFileSync(path.join(paths.TRANSLATIONS_DIR, `${lang}.json`), 'utf8'); - if (!translationData) debug(`getTranslations: Requested language ${lang} not found. ${safe.error.message}`); + if (!translationData) log(`getTranslations: Requested language ${lang} not found. ${safe.error.message}`); const translations = safe.JSON.parse(translationData) || {}; return { translations, fallback }; @@ -60,7 +60,7 @@ async function getTranslations() { async function listLanguages() { const [error, result] = await safe(fs.promises.readdir(paths.TRANSLATIONS_DIR)); if (error) { - debug(`listLanguages: Failed to list translations. %${error.message}`); + log(`listLanguages: Failed to list translations. %${error.message}`); return [ 'en' ]; // we always return english to avoid dashboard breakage } diff --git a/src/updater.js b/src/updater.js index 670434e39..66cdc9725 100644 --- a/src/updater.js +++ b/src/updater.js @@ -10,7 +10,7 @@ import constants from './constants.js'; import cron from './cron.js'; import { CronTime } from 'cron'; import crypto from 'node:crypto'; -import debugModule from 'debug'; +import logger from './logger.js'; import df from './df.js'; import eventlog from './eventlog.js'; import fs from 'node:fs'; @@ -26,7 +26,7 @@ import settings from './settings.js'; import shellModule from './shell.js'; import tasks from './tasks.js'; -const debug = debugModule('box:updater'); +const { log, trace } = logger('updater'); const shell = shellModule('updater'); @@ -58,11 +58,11 @@ async function downloadBoxUrl(url, file) { safe.fs.unlinkSync(file); - await promiseRetry({ times: 10, interval: 5000, debug }, async function () { - debug(`downloadBoxUrl: downloading ${url} to ${file}`); + await promiseRetry({ times: 10, interval: 5000, debug: log }, async function () { + log(`downloadBoxUrl: downloading ${url} to ${file}`); const [error] = await safe(shell.spawn('curl', ['-s', '--fail', url, '-o', file], { encoding: 'utf8' })); if (error) throw new BoxError(BoxError.NETWORK_ERROR, `Failed to download ${url}: ${error.message}`); - debug('downloadBoxUrl: done'); + log('downloadBoxUrl: done'); }); } @@ -72,13 +72,13 @@ async function gpgVerifyBoxTarball(file, sig) { const [error, stdout] = await safe(shell.spawn('/usr/bin/gpg', ['--status-fd', '1', '--no-default-keyring', '--keyring', RELEASES_PUBLIC_KEY, '--verify', sig, file], { encoding: 'utf8' })); if (error) { - debug(`gpgVerifyBoxTarball: command failed. error: ${error}\n stdout: ${error.stdout}\n stderr: ${error.stderr}`); + log(`gpgVerifyBoxTarball: command failed. error: ${error}\n stdout: ${error.stdout}\n stderr: ${error.stderr}`); throw new BoxError(BoxError.NOT_SIGNED, `The signature in ${path.basename(sig)} could not be verified (command failed)`); } if (stdout.indexOf('[GNUPG:] VALIDSIG 0EADB19CDDA23CD0FE71E3470A372F8703C493CC') !== -1) return; // success - debug(`gpgVerifyBoxTarball: verification of ${sig} failed: ${stdout}\n`); + log(`gpgVerifyBoxTarball: verification of ${sig} failed: ${stdout}\n`); throw new BoxError(BoxError.NOT_SIGNED, `The signature in ${path.basename(sig)} could not be verified (bad sig)`); } @@ -87,13 +87,13 @@ async function extractBoxTarball(tarball, dir) { assert.strictEqual(typeof tarball, 'string'); assert.strictEqual(typeof dir, 'string'); - debug(`extractBoxTarball: extracting ${tarball} to ${dir}`); + log(`extractBoxTarball: extracting ${tarball} to ${dir}`); const [error] = await safe(shell.spawn('tar', ['-zxf', tarball, '-C', dir], { encoding: 'utf8' })); if (error) throw new BoxError(BoxError.FS_ERROR, `Failed to extract release package: ${error.message}`); safe.fs.unlinkSync(tarball); - debug('extractBoxTarball: extracted'); + log('extractBoxTarball: extracted'); } async function verifyBoxUpdateInfo(versionsFile, updateInfo) { @@ -115,7 +115,7 @@ async function downloadAndVerifyBoxRelease(updateInfo) { const oldArtifactNames = filenames.filter(f => f.startsWith('box-')); for (const artifactName of oldArtifactNames) { const fullPath = path.join(os.tmpdir(), artifactName); - debug(`downloadAndVerifyBoxRelease: removing old artifact ${fullPath}`); + log(`downloadAndVerifyBoxRelease: removing old artifact ${fullPath}`); await fs.promises.rm(fullPath, { recursive: true, force: true }); } @@ -173,7 +173,7 @@ async function updateBox(boxUpdateInfo, options, progressCallback) { await locks.wait(locks.TYPE_BOX_UPDATE); - debug(`Updating box with ${boxUpdateInfo.sourceTarballUrl}`); + log(`Updating box with ${boxUpdateInfo.sourceTarballUrl}`); progressCallback({ percent: 70, message: 'Installing update...' }); const [error] = await safe(shell.sudo([ UPDATE_CMD, packageInfo.file, process.stdout.logFile ], {})); // run installer.sh from new box code as a separate service if (error) { @@ -226,9 +226,9 @@ async function startBoxUpdateTask(options, auditSource) { // background tasks.startTask(taskId, { timeout: 20 * 60 * 60 * 1000 /* 20 hours */, nice: 15, memoryLimit }) - .then(() => debug('startBoxUpdateTask: update task completed')) + .then(() => log('startBoxUpdateTask: update task completed')) .catch(async (updateError) => { - debug('Update failed with error. %o', updateError); + log('Update failed with error. %o', updateError); await locks.release(locks.TYPE_BOX_UPDATE_TASK); await locks.releaseByTaskId(taskId); @@ -249,7 +249,7 @@ async function notifyBoxUpdate() { if (!version) { await eventlog.add(eventlog.ACTION_INSTALL_FINISH, AuditSource.CRON, { version: constants.VERSION }); } else { - debug(`notifyBoxUpdate: update finished from ${version} to ${constants.VERSION}`); + log(`notifyBoxUpdate: update finished from ${version} to ${constants.VERSION}`); const [error] = await safe(tasks.setCompletedByType(tasks.TASK_BOX_UPDATE, { error: null })); if (error && error.reason !== BoxError.NOT_FOUND) throw error; // when hotfixing, task may not exist await eventlog.add(eventlog.ACTION_UPDATE_FINISH, AuditSource.CRON, { errorMessage: '', oldVersion: version || 'dev', newVersion: constants.VERSION }); @@ -265,10 +265,10 @@ async function autoUpdate(auditSource) { const boxUpdateInfo = await getBoxUpdate(); // do box before app updates. for the off chance that the box logic fixes some app update logic issue if (boxUpdateInfo && !boxUpdateInfo.unstable) { - debug('autoUpdate: starting box autoupdate to %j', boxUpdateInfo.version); + log('autoUpdate: starting box autoupdate to %j', boxUpdateInfo.version); const [error] = await safe(startBoxUpdateTask({ skipBackup: false }, AuditSource.CRON)); if (!error) return; // do not start app updates when a box update got scheduled - debug(`autoUpdate: failed to start box autoupdate task: ${error.message}`); + log(`autoUpdate: failed to start box autoupdate task: ${error.message}`); // fall through to update apps if box update never started (failed ubuntu or avx check) } @@ -276,13 +276,13 @@ async function autoUpdate(auditSource) { for (const app of result) { if (!app.updateInfo) continue; if (!app.updateInfo.isAutoUpdatable) { - debug(`autoUpdate: ${app.fqdn} requires manual update. skipping`); + log(`autoUpdate: ${app.fqdn} requires manual update. skipping`); continue; } const sites = await backupSites.listByContentForUpdates(app.id); if (sites.length === 0) { - debug(`autoUpdate: ${app.fqdn} has no backup site for updates. skipping`); + log(`autoUpdate: ${app.fqdn} has no backup site for updates. skipping`); continue; } @@ -291,9 +291,9 @@ async function autoUpdate(auditSource) { force: false }; - debug(`autoUpdate: ${app.fqdn} will be automatically updated`); + log(`autoUpdate: ${app.fqdn} will be automatically updated`); const [updateError] = await safe(apps.updateApp(app, data, auditSource)); - if (updateError) debug(`autoUpdate: error autoupdating ${app.fqdn}: ${updateError.message}`); + if (updateError) log(`autoUpdate: error autoupdating ${app.fqdn}: ${updateError.message}`); } } @@ -320,7 +320,7 @@ async function checkAppUpdate(app, options) { async function checkBoxUpdate(options) { assert.strictEqual(typeof options, 'object'); - debug('checkBoxUpdate: checking for updates'); + log('checkBoxUpdate: checking for updates'); const updateInfo = await appstore.getBoxUpdate(options); if (updateInfo) { @@ -346,7 +346,7 @@ async function raiseNotifications() { // currently, we do not raise notifications when auto-update is disabled. separate notifications appears spammy when having many apps // in the future, we can maybe aggregate? if (app.updateInfo && !app.updateInfo.isAutoUpdatable) { - debug(`autoUpdate: ${app.fqdn} cannot be autoupdated. skipping`); + log(`autoUpdate: ${app.fqdn} cannot be autoupdated. skipping`); await notifications.pin(notifications.TYPE_MANUAL_APP_UPDATE_NEEDED, `${app.manifest.title} at ${app.fqdn} requires manual update to version ${app.updateInfo.manifest.version}`, `Changelog:\n${app.updateInfo.manifest.changelog}\n`, { context: app.id }); continue; @@ -358,12 +358,12 @@ async function checkForUpdates(options) { assert.strictEqual(typeof options, 'object'); const [boxError] = await safe(checkBoxUpdate(options)); - if (boxError) debug('checkForUpdates: error checking for box updates: %o', boxError); + if (boxError) log('checkForUpdates: error checking for box updates: %o', boxError); // check app updates const result = await apps.list(); for (const app of result) { - await safe(checkAppUpdate(app, options), { debug }); + await safe(checkAppUpdate(app, options), { debug: log }); } // raise notifications here because the updatechecker runs regardless of auto-updater cron job diff --git a/src/user-directory.js b/src/user-directory.js index 3b07249f1..a2abdd4b8 100644 --- a/src/user-directory.js +++ b/src/user-directory.js @@ -1,7 +1,7 @@ import assert from 'node:assert'; import BoxError from './boxerror.js'; import constants from './constants.js'; -import debugModule from 'debug'; +import logger from './logger.js'; import eventlog from './eventlog.js'; import oidcClients from './oidcclients.js'; import oidcServer from './oidcserver.js'; @@ -9,7 +9,7 @@ import settings from './settings.js'; import tokens from './tokens.js'; import users from './users.js'; -const debug = debugModule('box:user-directory'); +const { log, trace } = logger('user-directory'); async function getProfileConfig() { @@ -30,7 +30,7 @@ async function setProfileConfig(profileConfig, options, auditSource) { await eventlog.add(eventlog.ACTION_USER_DIRECTORY_PROFILE_CONFIG_UPDATE, auditSource, { oldConfig, config: profileConfig }); if (profileConfig.mandatory2FA && !oldConfig.mandatory2FA) { - debug('setProfileConfig: logging out non-2FA users to enforce 2FA'); + log('setProfileConfig: logging out non-2FA users to enforce 2FA'); const allUsers = await users.list(); diff --git a/src/users.js b/src/users.js index a0e3f85a7..f646ab1a8 100644 --- a/src/users.js +++ b/src/users.js @@ -6,7 +6,7 @@ import constants from './constants.js'; import dashboard from './dashboard.js'; import branding from './branding.js'; import database from './database.js'; -import debugModule from 'debug'; +import logger from './logger.js'; import eventlog from './eventlog.js'; import externalLdap from './externalldap.js'; import hat from './hat.js'; @@ -30,7 +30,7 @@ import util from 'node:util'; import validator from './validator.js'; import _ from './underscore.js'; -const debug = debugModule('box:user'); +const { log, trace } = logger('user'); const AP_MAIL = 'mail'; const AP_WEBADMIN = 'webadmin'; @@ -534,20 +534,20 @@ async function verifyGhost(username, password) { if (username in ghostData) { if (typeof ghostData[username] === 'object') { if (ghostData[username].expiresAt < Date.now()) { - debug('verifyGhost: password expired'); + log('verifyGhost: password expired'); delete ghostData[username]; await settings.setJson(settings.GHOSTS_CONFIG_KEY, ghostData); return false; } else if (ghostData[username].password === password) { - debug('verifyGhost: matched ghost user'); + log('verifyGhost: matched ghost user'); return true; } else { return false; } } else if(ghostData[username] === password) { - debug('verifyGhost: matched ghost user'); + log('verifyGhost: matched ghost user'); delete ghostData[username]; await settings.setJson(settings.GHOSTS_CONFIG_KEY, ghostData); @@ -593,7 +593,7 @@ async function verify(user, password, identifier, options) { assert.strictEqual(typeof options, 'object'); if (!user.active) { - debug(`verify: ${user.username} is not active`); + log(`verify: ${user.username} is not active`); throw new BoxError(BoxError.NOT_FOUND, 'User not active'); } @@ -602,7 +602,7 @@ async function verify(user, password, identifier, options) { const valid = await verifyGhost(user.username, password); if (valid) { - debug(`verify: ${user.username} authenticated via impersonation`); + log(`verify: ${user.username} authenticated via impersonation`); user.ghost = true; return user; } @@ -610,14 +610,14 @@ async function verify(user, password, identifier, options) { const [error] = await safe(verifyAppPassword(user.id, password, identifier)); if (!error) { // matched app password - debug(`verify: ${user.username || user.id} matched app password`); + log(`verify: ${user.username || user.id} matched app password`); user.appPassword = true; return user; } const [mailPasswordError] = await safe(verifyMailPassword(user.id, password)); if (!mailPasswordError) { // matched app password - debug(`verify: ${user.username || user.id} matched mail password`); + log(`verify: ${user.username || user.id} matched mail password`); user.mailPassword = true; return user; } @@ -634,7 +634,7 @@ async function verify(user, password, identifier, options) { const derivedKeyHex = Buffer.from(derivedKey, 'binary').toString('hex'); if (derivedKeyHex !== user.password) { - debug(`verify: ${user.username || user.id} provided incorrect password`); + log(`verify: ${user.username || user.id} provided incorrect password`); throw new BoxError(BoxError.INVALID_CREDENTIALS, 'Wrong password'); } @@ -658,7 +658,7 @@ async function verifyWithId(id, password, identifier, options) { const user = await get(id); if (!user) { - debug(`verifyWithId: ${id} not found`); + log(`verifyWithId: ${id} not found`); throw new BoxError(BoxError.NOT_FOUND, 'User not found'); } @@ -676,11 +676,11 @@ async function verifyWithUsername(username, password, identifier, options) { const [error, newUserId] = await safe(externalLdap.maybeCreateUser(username.toLowerCase())); if (error && error.reason === BoxError.BAD_STATE) { - debug(`verifyWithUsername: ${username} not found`); + log(`verifyWithUsername: ${username} not found`); throw new BoxError(BoxError.NOT_FOUND, 'User not found'); // no external ldap or no auto create } if (error) { - debug(`verifyWithUsername: failed to auto create user ${username}. %o`, error); + log(`verifyWithUsername: failed to auto create user ${username}. %o`, error); throw new BoxError(BoxError.NOT_FOUND, 'User not found'); } @@ -695,7 +695,7 @@ async function verifyWithEmail(email, password, identifier, options) { const user = await getByEmail(email.toLowerCase()); if (!user) { - debug(`verifyWithEmail: ${email} no such user`); + log(`verifyWithEmail: ${email} no such user`); throw new BoxError(BoxError.NOT_FOUND, 'User not found'); } @@ -762,14 +762,14 @@ async function notifyLoginLocation(user, ip, userAgent, auditSource) { assert.strictEqual(typeof userAgent, 'string'); assert.strictEqual(typeof auditSource, 'object'); - debug(`notifyLoginLocation: ${user.id} ${ip} ${userAgent} ghost:${!!user.ghost}`); + log(`notifyLoginLocation: ${user.id} ${ip} ${userAgent} ghost:${!!user.ghost}`); if (constants.DEMO) return; if (constants.TEST && ip === '127.0.0.1') return; if (user.ghost || user.source) return; // for external users, rely on the external source to send login notification to avoid dup login emails const response = await superagent.get('https://geolocation.cloudron.io/json').query({ ip }).ok(() => true); - if (response.status !== 200) return debug(`Failed to get geoip info. status: ${response.status}`); + if (response.status !== 200) return log(`Failed to get geoip info. status: ${response.status}`); const country = safe.query(response.body, 'country.names.en', ''); const city = safe.query(response.body, 'city.names.en', ''); @@ -795,7 +795,7 @@ async function notifyLoginLocation(user, ip, userAgent, auditSource) { await update(user, { loginLocations }, auditSource); - await safe(mailer.sendNewLoginLocation(user, newLoginLocation), { debug }); + await safe(mailer.sendNewLoginLocation(user, newLoginLocation), { debug: log }); } async function setPassword(user, newPassword, auditSource) { diff --git a/src/volumes.js b/src/volumes.js index f6eb31f26..0ee1b1f13 100644 --- a/src/volumes.js +++ b/src/volumes.js @@ -2,7 +2,7 @@ import assert from 'node:assert'; import BoxError from './boxerror.js'; import crypto from 'node:crypto'; import database from './database.js'; -import debugModule from 'debug'; +import logger from './logger.js'; import eventlog from './eventlog.js'; import mounts from './mounts.js'; import path from 'node:path'; @@ -10,7 +10,7 @@ import paths from './paths.js'; import safe from 'safetydance'; import services from './services.js'; -const debug = debugModule('box:volumes'); +const { log, trace } = logger('volumes'); const VOLUMES_FIELDS = [ 'id', 'name', 'hostPath', 'creationTime', 'mountType', 'mountOptionsJson' ].join(','); @@ -87,7 +87,7 @@ async function add(volume, auditSource) { await eventlog.add(eventlog.ACTION_VOLUME_ADD, auditSource, { id, name, hostPath }); // in theory, we only need to do this mountpoint volumes. but for some reason a restart is required to detect new "mounts" - safe(services.rebuildService('sftp', auditSource), { debug }); + safe(services.rebuildService('sftp', auditSource), { debug: log }); return id; } @@ -166,7 +166,7 @@ async function update(volume, mountOptions, auditSource) { await database.query('UPDATE volumes SET hostPath=?,mountOptionsJson=? WHERE id=?', [ hostPath, JSON.stringify(mountOptions), volume.id ]); await eventlog.add(eventlog.ACTION_VOLUME_UPDATE, auditSource, { id: volume.id, name, hostPath }); // in theory, we only need to do this mountpoint volumes. but for some reason a restart is required to detect new "mounts" - safe(services.rebuildService('sftp', auditSource), { debug }); + safe(services.rebuildService('sftp', auditSource), { debug: log }); } async function list() { @@ -187,14 +187,14 @@ async function del(volume, auditSource) { await eventlog.add(eventlog.ACTION_VOLUME_REMOVE, auditSource, { ...volume }); if (volume.mountType === mounts.MOUNT_TYPE_MOUNTPOINT || volume.mountType === mounts.MOUNT_TYPE_FILESYSTEM) { - safe(services.rebuildService('sftp', auditSource), { debug }); + safe(services.rebuildService('sftp', auditSource), { debug: log }); } else { await safe(mounts.removeMount(volume)); } } async function mountAll() { - debug('mountAll: mouting all volumes'); + log('mountAll: mouting all volumes'); for (const volume of await list()) { if (volume.mountType === mounts.MOUNT_TYPE_MOUNTPOINT || volume.mountType === mounts.MOUNT_TYPE_FILESYSTEM) continue; diff --git a/syslog.js b/syslog.js index b1f452fc7..896db9610 100755 --- a/syslog.js +++ b/syslog.js @@ -1,13 +1,13 @@ #!/usr/bin/env node -import debugModule from 'debug'; +import logger from './src/logger.js'; import fs from 'node:fs'; import net from 'node:net'; import path from 'node:path'; import paths from './src/paths.js'; import util from 'node:util'; -const debug = debugModule('syslog:server'); +const { log, trace } = logger('syslog'); let gServer = null; @@ -37,9 +37,9 @@ function parseRFC5424Message(rawMessage) { } async function start() { - debug('=========================================='); - debug(' Cloudron Syslog Daemon '); - debug('=========================================='); + log('=========================================='); + log(' Cloudron Syslog Daemon '); + log('=========================================='); gServer = net.createServer(); @@ -52,8 +52,8 @@ async function start() { const msg = data.toString('utf8').trim(); // strip any trailing empty new lines. it's unclear why we get it in the first place for (const line of msg.split('\n')) { // empirically, multiple messages can arrive in a single packet const info = parseRFC5424Message(line); - if (!info) return debug(`Unable to parse: [${msg}]`); - if (!info.appName) return debug(`Ignore unknown app: [${msg}]`); + if (!info) return log(`Unable to parse: [${msg}]`); + if (!info.appName) return log(`Ignore unknown app: [${msg}]`); const appLogDir = path.join(paths.LOG_DIR, info.appName); @@ -61,20 +61,20 @@ async function start() { fs.mkdirSync(appLogDir, { recursive: true }); fs.appendFileSync(`${appLogDir}/app.log`, `${info.timestamp} ${info.message.trim()}\n`); } catch (error) { - debug(error); + log(error); } } }); socket.on('error', function (error) { - debug(`socket error: ${error}`); + log(`socket error: ${error}`); }); }); await fs.promises.rm(paths.SYSLOG_SOCKET_FILE, { force: true }); await util.promisify(gServer.listen.bind(gServer))(paths.SYSLOG_SOCKET_FILE); - debug(`Listening on ${paths.SYSLOG_SOCKET_FILE}`); + log(`Listening on ${paths.SYSLOG_SOCKET_FILE}`); } async function stop() { @@ -87,7 +87,7 @@ async function main() { await start(); process.on('SIGTERM', async function () { - debug('Received SIGTERM. Shutting down.'); + log('Received SIGTERM. Shutting down.'); await stop(); setTimeout(process.exit.bind(process), 1000); });