more 7.3.5 changes

cname record can be external and the original NS may not respond to
recursive queries

(cherry picked from commit e4d9dbb558)

CHANGES

@@ -2573,4 +2573,14 @@
 [7.3.4]
 * Display platform update status in the UI
 * Fix image pruning
+* cloudflare: fix issue where incorrect URL configuration is accepted
+
+[7.3.5]
+* du: fix crash when filesystem is cifs/nfs/sshfs
+* Start with a default to not fail if no swap is present
+* Fix bug in cert cleanup logic causing it to repeatedly cleanup
+* Fix crash in RBL check
+* unbound: disable controller interface explicitly
+* Fix issue where cert renewal logs where not displayed
+* Fix loading of mailboxes
 

package.json

@@ -17,7 +17,7 @@
     "aws-sdk": "^2.1248.0",
     "basic-auth": "^2.0.1",
     "body-parser": "^1.20.1",
-    "cloudron-manifestformat": "^5.19.0",
+    "cloudron-manifestformat": "^5.19.1",
     "connect": "^3.7.0",
     "connect-lastmile": "^2.1.1",
     "connect-timeout": "^1.9.0",

scripts/cloudron-setup

@@ -236,8 +236,8 @@ while true; do
     sleep 10
 done
 
-ip4=$(curl -s --fail --connect-timeout 2 --max-time 2 https://ipv4.api.cloudron.io/api/v1/helper/public_ip | sed -n -e 's/.*"ip": "\(.*\)"/\1/p' || true)
-ip6=$(curl -s --fail --connect-timeout 2 --max-time 2 https://ipv6.api.cloudron.io/api/v1/helper/public_ip | sed -n -e 's/.*"ip": "\(.*\)"/\1/p' || true)
+ip4=$(curl -s --fail --connect-timeout 10 --max-time 10 https://ipv4.api.cloudron.io/api/v1/helper/public_ip | sed -n -e 's/.*"ip": "\(.*\)"/\1/p' || true)
+ip6=$(curl -s --fail --connect-timeout 10 --max-time 10 https://ipv6.api.cloudron.io/api/v1/helper/public_ip | sed -n -e 's/.*"ip": "\(.*\)"/\1/p' || true)
 
 url4=""
 url6=""

scripts/init-ubuntu.sh

@@ -179,8 +179,8 @@ systemctl disable systemd-resolved || true
 # on vultr, ufw is enabled by default. we have our own firewall
 ufw disable || true
 
-# we need unbound to work as this is required for installer.sh to do any DNS requests
-echo -e "server:\n\tinterface: 127.0.0.1\n" > /etc/unbound/unbound.conf.d/cloudron-network.conf
+# we need unbound to work as this is required for installer.sh to do any DNS requests. control-enable is for https://github.com/NLnetLabs/unbound/issues/806
+echo -e "server:\n\tinterface: 127.0.0.1\n\nremote-control:\n\tcontrol-enable: no\n" > /etc/unbound/unbound.conf.d/cloudron-network.conf
 systemctl restart unbound
 
 # Ubuntu 22 has private home directories by default (https://discourse.ubuntu.com/t/private-home-directories-for-ubuntu-21-04-onwards/)

setup/start/unbound.conf

@@ -14,3 +14,8 @@ server:
         # enable below for logging to journalctl -u unbound
         # verbosity: 5
         # log-queries: yes
+
+# https://github.com/NLnetLabs/unbound/issues/806
+remote-control:
+    control-enable: no
+

src/backuptask.js

@@ -229,7 +229,11 @@ async function copy(backupConfig, srcRemotePath, destRemotePath, progressCallbac
     const newFilePath = backupFormat.api(format).getBackupFilePath(backupConfig, destRemotePath);
 
     const startTime = new Date();
-    await safe(storage.api(provider).copy(backupConfig, oldFilePath, newFilePath, progressCallback));
+    const [copyError] = await safe(storage.api(provider).copy(backupConfig, oldFilePath, newFilePath, progressCallback));
+    if (copyError) {
+        debug(`copy: copied to ${destRemotePath} errored. error: ${copyError.message}`);
+        throw copyError;
+    }
     debug(`copy: copied successfully to ${destRemotePath}. Took ${(new Date() - startTime)/1000} seconds`);
 }
 

src/constants.js

@@ -40,6 +40,7 @@ exports = module.exports = {
 
     DEMO_USERNAME: 'cloudron',
     DEMO_BLACKLISTED_APPS: [
+        'org.jupyter.cloudronapp',
         'com.github.cloudtorrent',
         'net.alltubedownload.cloudronapp',
         'com.adguard.home.cloudronapp',

src/dns/waitfordns.js

@@ -7,7 +7,8 @@ const assert = require('assert'),
     debug = require('debug')('box:dns/waitfordns'),
     dig = require('../dig.js'),
     promiseRetry = require('../promise-retry.js'),
-    safe = require('safetydance');
+    safe = require('safetydance'),
+    _ = require('underscore');
 
 async function resolveIp(hostname, type, options) {
     assert.strictEqual(typeof hostname, 'string');
@@ -20,12 +21,12 @@ async function resolveIp(hostname, type, options) {
     if (!error && results.length !== 0) return results;
 
     // try CNAME record at authoritative server
-    debug(`resolveIp: Checking if ${hostname} has CNAME record at ${options.server}`);
+    debug(`resolveIp: No A record. Checking if ${hostname} has CNAME record at ${options.server}`);
     const cnameResults = await dig.resolve(hostname, 'CNAME', options);
     if (cnameResults.length === 0) return cnameResults;
 
     // recurse lookup the CNAME record
-    debug(`resolveIp: Resolving ${hostname}'s CNAME record ${cnameResults[0]}`);
+    debug(`resolveIp: CNAME record found. Resolving ${hostname}'s CNAME record ${cnameResults[0]} using unbound`);
     return await dig.resolve(cnameResults[0], type, options);
 }
 

src/mail.js

@@ -539,7 +539,7 @@ async function checkRblStatus(domain) {
         const [error2, txtRecords] = await safe(dig.resolve(flippedIp + '.' + rblServer.dns, 'TXT', DNS_OPTIONS));
         result.txtRecords = error2 || !txtRecords ? 'No txt record' : txtRecords.map(x => x.join(''));
 
-        debug(`checkRblStatus: ${domain} (error: ${error2.message}) (txtRecords: ${JSON.stringify(txtRecords)})`);
+        debug(`checkRblStatus: ${domain} (error: ${error2?.message || null}) (txtRecords: ${JSON.stringify(txtRecords)})`);
 
         blacklistedServers.push(result);
     }
@@ -590,7 +590,7 @@ async function getStatus(domain) {
     for (let i = 0; i < checks.length; i++) {
         const response = responses[i], check = checks[i];
         if (response.status !== 'fulfilled') {
-            debug(`check ${check.what} was rejected. This is not expected`);
+            debug(`check ${check.what} was rejected. This is not expected. reason: ${response.reason}`);
             continue;
         }
 

src/reverseproxy.js

@@ -592,7 +592,7 @@ async function cleanupCerts(locations, auditSource, progressCallback) {
     const certIds = await blobs.listCertIds();
     const removedCertNames = [];
     for (const certId of certIds) {
-        const certName = certId.match(new RegExp(`${blobs.CERT_PREFIX}-(.*).cert`))[0];
+        const certName = certId.match(new RegExp(`${blobs.CERT_PREFIX}-(.*).cert`))[1];
         if (certNamesInUse.has(certName)) continue;
 
         const cert = await blobs.getString(certId);

src/storage/s3.js

@@ -454,6 +454,7 @@ async function copy(apiConfig, oldFilePath, newFilePath, progressCallback) {
     }));
 
     progressCallback({ message: `Copied ${total} files with error: ${copyError}` });
+    if (copyError) throw copyError;
 }
 
 async function remove(apiConfig, filename) {
@@ -560,7 +561,7 @@ async function testConfig(apiConfig) {
 
     const putParams = {
         Bucket: apiConfig.bucket,
-        Key: path.join(apiConfig.prefix, 'cloudron-testfile'),
+        Key: path.join(apiConfig.prefix, 'snapshot/cloudron-testfile'),
         Body: 'testcontent'
     };
 
@@ -568,9 +569,18 @@ async function testConfig(apiConfig) {
     const [putError] = await safe(s3.putObject(putParams).promise());
     if (putError) throw new BoxError(BoxError.EXTERNAL_ERROR, `Error put object cloudron-testfile. Message: ${putError.message} HTTP Code: ${putError.code}`);
 
+    const listParams = {
+        Bucket: apiConfig.bucket,
+        Prefix: path.join(apiConfig.prefix, 'snapshot'),
+        MaxKeys: 1
+    };
+
+    const [listError] = await safe(s3.listObjects(listParams).promise());
+    if (listError) throw new BoxError(BoxError.EXTERNAL_ERROR, `Error listing objects. Message: ${listError.message} HTTP Code: ${listError.code}`);
+
     const delParams = {
         Bucket: apiConfig.bucket,
-        Key: path.join(apiConfig.prefix, 'cloudron-testfile')
+        Key: path.join(apiConfig.prefix, 'snapshot/cloudron-testfile')
     };
 
     const [delError] = await safe(s3.deleteObject(delParams).promise());

src/system.js

@@ -62,8 +62,10 @@ async function getDisks() {
     const disks = {}; // by file system
     let rootDisk;
 
+    const DISK_TYPES = [ 'ext4', 'xfs', 'cifs', 'nfs', 'fuse.sshfs' ]; // we don't show size of contents in untracked disk types
+
     for (const disk of dfEntries) {
-        if (disk.type !== 'ext4' && disk.type !== 'xfs') continue;
+        if (!DISK_TYPES.includes(disk.type)) continue;
         if (disk.mountpoint === '/') rootDisk = disk;
         disks[disk.filesystem] = {
             filesystem: disk.filesystem,
@@ -92,18 +94,21 @@ async function getDisks() {
     const backupConfig = await settings.getBackupConfig();
     if (backupConfig.provider === 'filesystem') {
         const [, dfResult] = await safe(df.file(backupConfig.backupFolder));
-        disks[dfResult?.filesystem || rootDisk.filesystem].contents.push({ type: 'standard', id: 'cloudron-backup', path: backupConfig.backupFolder });
+        const filesystem = dfResult?.filesystem || rootDisk.filesystem;
+        if (disks[filesystem]) disks[filesystem].contents.push({ type: 'standard', id: 'cloudron-backup', path: backupConfig.backupFolder });
     }
 
     const [dockerError, dockerInfo] = await safe(docker.info());
     if (!dockerError) {
         const [, dfResult] = await safe(df.file(dockerInfo.DockerRootDir));
-        disks[dfResult?.filesystem || rootDisk.filesystem].contents.push({ type: 'standard', id: 'docker', path: dockerInfo.DockerRootDir });
+        const filesystem = dfResult?.filesystem || rootDisk.filesystem;
+        if (disks[filesystem]) disks[filesystem].contents.push({ type: 'standard', id: 'docker', path: dockerInfo.DockerRootDir });
     }
 
     for (const volume of await volumes.list()) {
         const [, dfResult] = await safe(df.file(volume.hostPath));
-        disks[dfResult?.filesystem || rootDisk.filesystem].contents.push({ type: 'volume', id: volume.id, path: volume.hostPath });
+        const filesystem = dfResult?.filesystem || rootDisk.filesystem;
+        if (disks[filesystem]) disks[filesystem].contents.push({ type: 'volume', id: volume.id, path: volume.hostPath });
     }
 
     for (const app of await apps.list()) {
@@ -111,7 +116,8 @@ async function getDisks() {
 
         const dataDir = await apps.getStorageDir(app);
         const [, dfResult] = await safe(df.file(dataDir));
-        disks[dfResult?.filesystem || rootDisk.filesystem].contents.push({ type: 'app', id: app.id, path: dataDir });
+        const filesystem = dfResult?.filesystem || rootDisk.filesystem;
+        if (disks[filesystem]) disks[filesystem].contents.push({ type: 'app', id: app.id, path: dataDir });
     }
 
     const swaps = await getSwaps();
@@ -151,7 +157,7 @@ async function checkDiskSpace() {
 
 async function getSwapSize() {
     const swaps = await getSwaps();
-    return Object.keys(swaps).map(n => swaps[n].size).reduce((acc, cur) => acc + cur);
+    return Object.keys(swaps).map(n => swaps[n].size).reduce((acc, cur) => acc + cur, 0);
 }
 
 async function getMemory() {