firewall: add dockerproxy

this helps in updating rules across upgrades

CHANGES

@@ -2882,3 +2882,20 @@
 * postgres: enable vector extension
 * docker: fallback to downloading images from quay if dockerhub does not work
 
+[8.2.1]
+* apps: fix bug where update and notes indicator was shown to normal users
+* archive: disable archiving for pre-8.2 backups. we don't have enough info to unarchive
+* dashboard: fix browser caching issue
+
+[8.2.2]
+* gandi: add token type in the setup view
+* mail: fix issue with dkim signing
+* mail: fix crash in dns list plugin
+* scheduler: create jobs with cloudron tz setting
+* security: fix issue where '/' symlink allows admins to get ssh access
+
+[8.2.3]
+* mail: give container a static IP
+* firewall: add masquerading rules for containers to reach each other via public IP
+* docker: fix parsing of optional namespace in image refs
+

README.md

@@ -45,20 +45,19 @@ Try our demo at https://my.demo.cloudron.io (username: cloudron password: cloudr
 
 [Install script](https://docs.cloudron.io/installation/) - [Pricing](https://cloudron.io/pricing.html)
 
-**Note:** This repo is a small part of what gets installed on your server - there is
-the dashboard, database addons, graph container, base image etc. Cloudron also relies
-on external services such as the App Store for apps to be installed. As such, don't
-clone this repo and npm install and expect something to work.
+**Note:** This repo is just a part of what gets installed on the server. Database addons,
+Mail Server, Stat contains etc are not part of this repo. As such, don't clone this repo and
+npm install and expect something to work.
 
 ## License
 
 Please note that the Cloudron code is under a source-available license. This is not the same as an
-open source license but ensures the code is available for introspection (and hacking!).
+open source license but ensures the code is available for transparency and introspection (and hacking!).
 
 ## Contributions
 
-Just to give some heads up, we are a bit restrictive in merging changes. We are a small team and
-would like to keep our maintenance burden low. It might be best to discuss features first in the [forum](https://forum.cloudron.io),
+We are very restrictive in merging changes. We are a small team and would like to keep our maintenance burden low,
+not to mention legal issues. It might be best to discuss features first in the [forum](https://forum.cloudron.io),
 to also figure out how many other people will use it to justify maintenance for a feature.
 
 # Localization

dashboard/index.html

@@ -63,6 +63,8 @@
   <script type="text/javascript" src="/js/timezones.js?%VITE_CACHE_ID%"></script>
 
   <!-- Main Application -->
+  <!-- for now we need this in a static non transformed index.js file -->
+  <script> window.VITE_CACHE_ID = '%VITE_CACHE_ID%' </script>
   <script type="text/javascript" src="/js/index.js?%VITE_CACHE_ID%"></script>
   <script type="text/javascript" src="/js/client.js?%VITE_CACHE_ID%"></script>
   <script type="text/javascript" src="/js/utils.js?%VITE_CACHE_ID%"></script>

dashboard/public/js/index.js

@@ -48,72 +48,72 @@ app.config(['$routeProvider', function ($routeProvider) {
         redirectTo: '/apps'
     }).when('/users', {
         controller: 'UsersController',
-        templateUrl: 'views/users.html?<%= revision %>'
+        templateUrl: 'views/users.html?' + window.VITE_CACHE_ID
     }).when('/user-directory', {
         controller: 'UserSettingsController',
-        templateUrl: 'views/user-directory.html?<%= revision %>'
+        templateUrl: 'views/user-directory.html?' + window.VITE_CACHE_ID
     }).when('/app/:appId/:view?', {
         controller: 'AppController',
-        templateUrl: 'views/app.html?<%= revision %>'
+        templateUrl: 'views/app.html?' + window.VITE_CACHE_ID
     }).when('/appstore', {
         controller: 'AppStoreController',
-        templateUrl: 'views/appstore.html?<%= revision %>'
+        templateUrl: 'views/appstore.html?' + window.VITE_CACHE_ID
     }).when('/appstore/:appId', {
         controller: 'AppStoreController',
-        templateUrl: 'views/appstore.html?<%= revision %>'
+        templateUrl: 'views/appstore.html?' + window.VITE_CACHE_ID
     }).when('/apps', {
         controller: 'AppsController',
-        templateUrl: 'views/apps.html?<%= revision %>'
+        templateUrl: 'views/apps.html?' + window.VITE_CACHE_ID
     }).when('/profile', {
         controller: 'ProfileController',
-        templateUrl: 'views/profile.html?<%= revision %>'
+        templateUrl: 'views/profile.html?' + window.VITE_CACHE_ID
     }).when('/backups', {
         controller: 'BackupsController',
-        templateUrl: 'views/backups.html?<%= revision %>'
+        templateUrl: 'views/backups.html?' + window.VITE_CACHE_ID
     }).when('/branding', {
         controller: 'BrandingController',
-        templateUrl: 'views/branding.html?<%= revision %>'
+        templateUrl: 'views/branding.html?' + window.VITE_CACHE_ID
     }).when('/network', {
         controller: 'NetworkController',
-        templateUrl: 'views/network.html?<%= revision %>'
+        templateUrl: 'views/network.html?' + window.VITE_CACHE_ID
     }).when('/domains', {
         controller: 'DomainsController',
-        templateUrl: 'views/domains.html?<%= revision %>'
+        templateUrl: 'views/domains.html?' + window.VITE_CACHE_ID
     }).when('/email', {
         controller: 'EmailsController',
-        templateUrl: 'views/emails.html?<%= revision %>'
+        templateUrl: 'views/emails.html?' + window.VITE_CACHE_ID
     }).when('/emails-eventlog', {
         controller: 'EmailsEventlogController',
-        templateUrl: 'views/emails-eventlog.html?<%= revision %>'
+        templateUrl: 'views/emails-eventlog.html?' + window.VITE_CACHE_ID
     }).when('/emails-queue', {
         controller: 'EmailsQueueController',
-        templateUrl: 'views/emails-queue.html?<%= revision %>'
+        templateUrl: 'views/emails-queue.html?' + window.VITE_CACHE_ID
     }).when('/email/:domain/:view?', {
         controller: 'EmailController',
-        templateUrl: 'views/email.html?<%= revision %>'
+        templateUrl: 'views/email.html?' + window.VITE_CACHE_ID
     }).when('/notifications', {
         controller: 'NotificationsController',
-        templateUrl: 'views/notifications.html?<%= revision %>'
+        templateUrl: 'views/notifications.html?' + window.VITE_CACHE_ID
     }).when('/oidc', {
         redirectTo: '/user-directory'
     }).when('/settings', {
         controller: 'SettingsController',
-        templateUrl: 'views/settings.html?<%= revision %>'
+        templateUrl: 'views/settings.html?' + window.VITE_CACHE_ID
     }).when('/eventlog', {
         controller: 'EventLogController',
-        templateUrl: 'views/eventlog.html?<%= revision %>'
+        templateUrl: 'views/eventlog.html?' + window.VITE_CACHE_ID
     }).when('/support', {
         controller: 'SupportController',
-        templateUrl: 'views/support.html?<%= revision %>'
+        templateUrl: 'views/support.html?' + window.VITE_CACHE_ID
     }).when('/system', {
         controller: 'SystemController',
-        templateUrl: 'views/system.html?<%= revision %>'
+        templateUrl: 'views/system.html?' + window.VITE_CACHE_ID
     }).when('/services', {
         controller: 'ServicesController',
-        templateUrl: 'views/services.html?<%= revision %>'
+        templateUrl: 'views/services.html?' + window.VITE_CACHE_ID
     }).when('/volumes', {
         controller: 'VolumesController',
-        templateUrl: 'views/volumes.html?<%= revision %>'
+        templateUrl: 'views/volumes.html?' + window.VITE_CACHE_ID
     }).otherwise({ redirectTo: '/'});
 }]);
 

dashboard/public/js/setup.js

@@ -87,6 +87,7 @@ app.controller('SetupDNSController', ['$scope', '$http', '$timeout', 'Client', f
         gcdnsKey: { keyFileName: '', content: '' },
         digitalOceanToken: '',
         gandiApiKey: '',
+        gandiTokenType: 'PAT',
         cloudflareEmail: '',
         cloudflareToken: '',
         cloudflareTokenType: 'GlobalApiKey',
@@ -181,6 +182,7 @@ app.controller('SetupDNSController', ['$scope', '$http', '$timeout', 'Client', f
             config.token = $scope.dnsCredentials.digitalOceanToken;
         } else if (provider === 'gandi') {
             config.token = $scope.dnsCredentials.gandiApiKey;
+            config.tokenType = $scope.dnsCredentials.gandiTokenType;
         } else if (provider === 'godaddy') {
             config.apiKey = $scope.dnsCredentials.godaddyApiKey;
             config.apiSecret = $scope.dnsCredentials.godaddyApiSecret;

dashboard/public/translation/en.json

@@ -1824,7 +1824,7 @@
             "description": "The latest app backup will be added to the <a href=\"/#backups\">App Archive</a>. The app will be uninstalled, but can be restored from the Backups View. Other backups will be cleaned up based on the backup policy.",
             "action": "Archive",
             "latestBackupInfo": "The last backup was created at {{date}}.",
-            "noBackup": "This app has no backup. Archiving requires at least one backup."
+            "noBackup": "This app has no backup. Archiving requires a recent backup."
         },
         "archiveDialog": {
             "title": "Archive {{app}}",

dashboard/public/views/app.js

@@ -1757,7 +1757,8 @@ angular.module('Application').controller('AppController', ['$scope', '$location'
             $scope.uninstall.latestBackup = null;
 
             Client.getAppBackups($scope.app.id, function (error, backups) {
-                if (!error && backups.length) $scope.uninstall.latestBackup = backups[0];
+                // only backups with appConfig (post 8.2) are candidates for archive
+                if (!error && backups.length) $scope.uninstall.latestBackup = backups[0].appConfig ? backups[0] : null;
             });
         },
 

dashboard/public/views/apps.html

@@ -201,7 +201,8 @@
           </div>
 
           <!-- we check the version here because the box updater does not know when an app gets updated -->
-          <div class="app-update-badge" ng-click="showAppConfigure(app, 'updates')" ng-show="config.update[app.id].manifest.version && config.update[app.id].manifest.version !== app.manifest.version && (app | installSuccess) && !(app.error || app.runState === 'stopped')" uib-tooltip="Update Available">
+          <!-- update info is available to app users. but we should show update indicator only for operators since normal users cannot update -->
+          <div class="app-update-badge" ng-click="showAppConfigure(app, 'updates')" ng-show="isOperator(app) && config.update[app.id].manifest.version && config.update[app.id].manifest.version !== app.manifest.version && (app | installSuccess) && !(app.error || app.runState === 'stopped')" uib-tooltip="Update Available">
             <i class="fa fa-arrow-up fa-inverse"></i>
           </div>
 

dashboard/public/views/apps.js

@@ -76,7 +76,12 @@ angular.module('Application').controller('AppsController', ['$scope', '$translat
     $scope.orderByFilter = function (item) {
         if ($scope.orderBy === 'app') return item.manifest.title || 'App Link';
         if ($scope.orderBy === 'status') return item.installationState + '-' + item.runState;
-        if ($scope.orderBy === 'sso') return item.sso;
+        if ($scope.orderBy === 'sso') {
+            if (item.ssoAuth && item.manifest.addons.oidc) return 'oidc';
+            if (item.ssoAuth && (!item.manifest.addons.oidc && !item.manifest.addons.email)) return 'sso';
+            if (item.manifest.addons.email) return 'email';
+            return '';
+        }
         return item.label || item.fqdn;
     };
 

dashboard/public/views/backups.html

@@ -458,7 +458,7 @@
         <h4 class="modal-title">{{ 'backups.restoreArchiveDialog.title' | tr }}</h4>
       </div>
       <div class="modal-body" style="padding: 0 15px">
-        <p ng-bind-html="'backups.restoreArchiveDialog.description' | tr:{ appId: archiveRestore.app.manifest.id, fqdn: archiveRestore.app.fqdn, creationTime: (archiveRestore.archive.creationTime | prettyLongDate) }"></p>
+        <p ng-bind-html="'backups.restoreArchiveDialog.description' | tr:{ appId: archiveRestore.manifest.id, fqdn: archiveRestore.fqdn, creationTime: (archiveRestore.archive.creationTime | prettyLongDate) }"></p>
         <form role="form" ng-submit="archiveRestore.submit()" autocomplete="off">
           <fieldset>
             <div class="form-group" ng-class="{ 'has-error': archiveRestore.error.location.fqdn === archiveRestore.subdomain + '.' + archiveRestore.domain.domain }">
@@ -481,7 +481,7 @@
             </div>
 
             <div class="has-error text-center" ng-show="archiveRestore.error.secondaryDomain">{{ archiveRestore.error.secondaryDomain }}</div>
-            <div ng-repeat="(env, info) in archiveRestore.app.manifest.httpPorts">
+            <div ng-repeat="(env, info) in archiveRestore.manifest.httpPorts">
               <ng-form name="secondaryDomainInfo_form">
                 <div class="form-group" ng-class="{ 'has-error': (!secondaryDomainInfo_form.itemName{{$index}}.$dirty && archiveRestore.error.secondaryDomain) || (secondaryDomainInfo_form.itemName{{$index}}.$dirty && secondaryDomainInfo_form.itemName{{$index}}.$invalid) || (archiveRestore.error.location.fqdn === archiveRestore.secondaryDomains[env].subdomain + '.' + archiveRestore.secondaryDomains[env].domain.domain) }">
                   <label class="control-label" for="secondaryDomainInput{{env}}">
@@ -535,7 +535,7 @@
 
       <div class="modal-footer">
         <button type="button" class="btn btn-default" data-dismiss="modal">{{ 'main.dialog.cancel' | tr }}</button>
-        <button type="button" class="btn btn-success" ng-click="archiveRestore.submit()"><i class="fas fa-history" ng-hide="archiveRestore.busy"></i><i class="fa fa-circle-notch fa-spin" ng-show="archiveRestore.busy"></i> {{ 'backups.restoreArchiveDialog.restoreAction' | tr:{ dnsOverwrite: archiveRestore.needsOverwrite } }}</button>
+        <button type="button" class="btn btn-success" ng-click="archiveRestore.submit()" ng-disabled="archiveRestore.busy"><i class="fas fa-history" ng-hide="archiveRestore.busy"></i><i class="fa fa-circle-notch fa-spin" ng-show="archiveRestore.busy"></i> {{ 'backups.restoreArchiveDialog.restoreAction' | tr:{ dnsOverwrite: archiveRestore.needsOverwrite } }}</button>
       </div>
     </div>
   </div>
@@ -546,7 +546,7 @@
   <div class="modal-dialog">
     <div class="modal-content">
       <div class="modal-header">
-        <h4 class="modal-title">{{ 'backups.deleteArchiveDialog.title' | tr:{ appTitle: archiveDelete.app.manifest.title, fqdn: archiveDelete.app.fqdn } }}</h4>
+        <h4 class="modal-title">{{ 'backups.deleteArchiveDialog.title' | tr:{ appTitle: archiveDelete.title, fqdn: archiveDelete.fqdn } }}</h4>
       </div>
       <div class="modal-body">
         <p>{{ 'backups.deleteArchiveDialog.description' | tr }}</p>
@@ -783,11 +783,12 @@
                 <td>
                   <img ng-src="{{ archive.iconUrl || 'img/appicon_fallback.png' }}" fallback-icon="img/appicon_fallback.png" onerror="imageErrorHandler(this)" height="48" width="48"/>
                 </td>
+                <!-- for pre-8.2 backups, appConfig can be null -->
                 <td class="hand elide-table-cell" style="text-overflow: ellipsis; white-space: nowrap;" ng-click="archiveRestore.show(archive)">
-                  {{ archive.appConfig.fqdn }}
+                  {{ archive.appConfig ? archive.appConfig.fqdn : '-' }}
                 </td>
                 <td class="hand elide-table-cell hide-mobile" style="text-overflow: ellipsis; white-space: nowrap;" ng-click="archiveRestore.show(archive)">
-                  <span uib-tooltip="{{ archive.appConfig.manifest.id }}@{{ archive.appConfig.manifest.version }}">{{ archive.appConfig.manifest.title }}</span>
+                  <span uib-tooltip="{{ archive.manifest.id }}@{{ archive.manifest.version }}">{{ archive.manifest.title }}</span>
                 </td>
                 <td class="hand elide-table-cell" style="text-overflow: ellipsis; white-space: nowrap;" ng-click="archiveRestore.show(archive)">
                   {{ archive.creationTime | prettyDate }}

dashboard/public/views/backups.js

@@ -290,13 +290,15 @@ angular.module('Application').controller('BackupsController', ['$scope', '$locat
         busy: false,
         error: {},
         archive: null,
-        app: null, // just for simpler access . it's a fake app object!
+        title: '',
+        fqdn: '',
 
         ask: function (archive) {
             $scope.archiveDelete.busy = false;
             $scope.archiveDelete.error = {};
             $scope.archiveDelete.archive = archive;
-            $scope.archiveDelete.app = archive.appConfig;
+            $scope.archiveDelete.title = archive.manifest.title;
+            $scope.archiveDelete.fqdn = archive.appConfig?.fqdn || '-';
             $('#archiveDeleteModal').modal('show');
         },
 
@@ -319,7 +321,9 @@ angular.module('Application').controller('BackupsController', ['$scope', '$locat
         error: {},
 
         archive: null,
-        app: null, // just for simpler access . it's a fake app object!
+        manifest: null,
+        appStoreId: '',
+        fqdn: '',
 
         subdomain: '',
         domain: null,
@@ -341,18 +345,26 @@ angular.module('Application').controller('BackupsController', ['$scope', '$locat
         show: function (archive) {
             $scope.archiveRestore.error = {};
             $scope.archiveRestore.archive = archive;
-            const manifest = archive.appConfig.manifest;
+            $scope.archiveRestore.manifest = archive.manifest;
 
-            $scope.archiveRestore.app = archive.appConfig;
-            $scope.archiveRestore.subdomain = $scope.archiveRestore.app.subdomain;
-            $scope.archiveRestore.domain = $scope.domains.find(function (d) { return $scope.archiveRestore.app.domain === d.domain; }); // try to pre-select the app's domain
+            const app = archive.appConfig || {
+                subdomain: '',
+                domain: $scope.domains[0].domain,
+                secondaryDomains: [],
+                portBindings: {}
+            }; // pre-8.2 backups do not have appConfig
+
+            $scope.archiveRestore.fqdn = archive.appConfig?.fqdn || '-';
+
+            $scope.archiveRestore.subdomain = app.subdomain;
+            $scope.archiveRestore.domain = $scope.domains.find(function (d) { return app.domain === d.domain; }); // try to pre-select the app's domain
 
             $scope.archiveRestore.needsOverwrite = false;
             $scope.archiveRestore.overwriteDns = false;
 
             $scope.archiveRestore.secondaryDomains = {};
 
-            var httpPorts = manifest.httpPorts || {};
+            var httpPorts = archive.manifest.httpPorts || {};
             for (var env2 in httpPorts) {
                 $scope.archiveRestore.secondaryDomains[env2] = {
                     subdomain: httpPorts[env2].defaultValue || '',
@@ -360,18 +372,18 @@ angular.module('Application').controller('BackupsController', ['$scope', '$locat
                 };
             }
             // now fill secondaryDomains with real values, if it exists
-            $scope.archiveRestore.app.secondaryDomains.forEach(function (sd) {
+            app.secondaryDomains.forEach(function (sd) {
                 $scope.archiveRestore.secondaryDomains[sd.environmentVariable] = {
                     subdomain: sd.subdomain,
                     domain: $scope.domains.find(function (d) { return sd.domain === d.domain; })
                 };
             });
 
-            $scope.archiveRestore.portInfo = angular.extend({}, manifest.tcpPorts, manifest.udpPorts);   // Portbinding map only for information
+            $scope.archiveRestore.portInfo = angular.extend({}, archive.manifest.tcpPorts, archive.manifest.udpPorts);   // Portbinding map only for information
             // set default ports
             for (var env in $scope.archiveRestore.portInfo) {
-                if ($scope.archiveRestore.app.portBindings[env]) { // was enabled in the app
-                    $scope.archiveRestore.ports[env] = $scope.archiveRestore.app.portBindings[env].hostPort;
+                if (app.portBindings[env]) { // was enabled in the app
+                    $scope.archiveRestore.ports[env] = app.portBindings[env].hostPort;
                     $scope.archiveRestore.portsEnabled[env] = true;
                 } else {
                     $scope.archiveRestore.ports[env] = $scope.archiveRestore.portInfo[env].defaultValue || 0;

dashboard/setup.html

@@ -146,6 +146,13 @@
                 </div>
 
                 <!-- Gandi -->
+                <div class="form-group" ng-class="{ 'has-error': false }" ng-show="dnsCredentials.provider === 'gandi'">
+                  <label class="control-label">Token Type</label>
+                  <select class="form-control" ng-model="dnsCredentials.gandiTokenType">
+                    <option value="ApiKey">API Key (Deprecated)</option>
+                    <option value="PAT">Personal Access Token (PAT)</option>
+                  </select>
+                </div>
                 <div class="form-group" ng-class="{ 'has-error': dnsCredentialsForm.gandiApiKey.$dirty && dnsCredentialsForm.gandiApiKey.$invalid }" ng-show="dnsCredentials.provider === 'gandi'">
                   <label class="control-label">Gandi API Key</label>
                   <input type="text" class="form-control" ng-model="dnsCredentials.gandiApiKey" name="gandiApiKey" placeholder="API Key" ng-required="dnsCredentials.provider === 'gandi'" ng-disabled="dnsCredentials.busy">

run-tests

@@ -24,7 +24,9 @@ cd ${DATA_DIR}
 mkdir -p appsdata
 mkdir -p boxdata/box boxdata/mail boxdata/certs boxdata/mail/dkim/localhost boxdata/mail/dkim/foobar.com
 mkdir -p platformdata/addons/mail/banner platformdata/nginx/cert platformdata/nginx/applications/dashboard platformdata/collectd/collectd.conf.d platformdata/addons platformdata/logrotate.d platformdata/backup platformdata/logs/tasks platformdata/sftp/ssh platformdata/firewall platformdata/update platformdata/diskusage
-sudo mkdir -p /mnt/cloudron-test-music /media/cloudron-test-music # volume test
+sudo mkdir -p /mnt/cloudron-test-music /media/cloudron-test-music /mnt/cloudron-test-music2  # volume test
+sudo ln -sf /root /media/cloudron-test-music/root
+sudo touch /media/cloudron-test-music/file
 
 # put cert
 echo "=> Generating a localhost selfsigned cert"

scripts/cloudron-support

@@ -18,6 +18,8 @@ readonly LINE="\n========================================================\n"
 readonly HELP_MESSAGE="
  Cloudron Support and Diagnostics Tool
  
+ See https://docs.cloudron.io/troubleshooting for more information on troubleshooting.
+
  Options:
    --disable-dnssec           Disable DNSSEC
    --enable-remote-support     Enable SSH Remote Access for the Cloudron support team

setup/start/cloudron-firewall.sh

@@ -7,8 +7,8 @@ echo "==> Setting up firewall"
 has_ipv6=$(cat /proc/net/if_inet6 >/dev/null 2>&1 && echo "yes" || echo "no")
 
 # wait for 120 seconds for xtables lock, checking every 1 second
-readonly iptables="iptables --wait 120 --wait-interval 1"
-readonly ip6tables="ip6tables --wait 120 --wait-interval 1"
+readonly iptables="iptables --wait 120"
+readonly ip6tables="ip6tables --wait 120"
 
 function ipxtables() {
     $iptables "$@"
@@ -110,7 +110,9 @@ $iptables -t filter -A CLOUDRON -p icmp --icmp-type echo-reply -j ACCEPT
 $ip6tables -t filter -A CLOUDRON -p ipv6-icmp -j ACCEPT
 
 ipxtables -t filter -A CLOUDRON -p udp --sport 53 -j ACCEPT
-$iptables -t filter -A CLOUDRON -s 172.18.0.0/16 -j ACCEPT # required to accept any connections from apps to our IP:<public port>
+# for ldap,dockerproxy server (ipv4 only) to accept connections from apps. for connecting to addons and mail container ports, docker already has rules
+$iptables -t filter -A CLOUDRON -p tcp -s 172.18.0.0/16 -d 172.18.0.1 -m multiport --dports 3002,3003 -j ACCEPT
+$iptables -t filter -A CLOUDRON -p udp -s 172.18.0.0/16 --dport 53 -j ACCEPT # dns responses from docker (127.0.0.11)
 ipxtables -t filter -A CLOUDRON -i lo -j ACCEPT # required for localhost connections (mysql)
 
 # log dropped incoming. keep this at the end of all the rules
@@ -118,10 +120,21 @@ ipxtables -t filter -A CLOUDRON -m limit --limit 2/min -j LOG --log-prefix "Pack
 ipxtables -t filter -A CLOUDRON -j DROP
 
 # prepend our chain to the filter table
-echo "==> Adding cloudron chain"
+echo "==> Adding cloudron filter chain"
 $iptables -t filter -C INPUT -j CLOUDRON 2>/dev/null || $iptables -t filter -I INPUT -j CLOUDRON
 $ip6tables -t filter -C INPUT -j CLOUDRON 2>/dev/null || $ip6tables -t filter -I INPUT -j CLOUDRON
 
+# masquerading rules for container ports to be accessible using public IP from other containers
+echo "==> Adding cloudron postrouting chain"
+ipxtables -t nat -N CLOUDRON_POSTROUTING || true
+ipxtables -t nat -F CLOUDRON_POSTROUTING # empty any existing rules
+
+$iptables -t nat -A CLOUDRON_POSTROUTING -s 172.18.0.0/16 -d 172.18.0.0/16 -j MASQUERADE
+$ip6tables -t nat -A CLOUDRON_POSTROUTING -s fd00:c107:d509::/64 -d fd00:c107:d509::/64 -j MASQUERADE
+
+$iptables -t nat -C POSTROUTING -j CLOUDRON_POSTROUTING 2>/dev/null || $iptables -t nat -I POSTROUTING -j CLOUDRON_POSTROUTING
+$ip6tables -t nat -C POSTROUTING -j CLOUDRON_POSTROUTING 2>/dev/null || $ip6tables -t nat -I POSTROUTING -j CLOUDRON_POSTROUTING
+
 # Setup rate limit chain (the recent info is at /proc/net/xt_recent)
 echo "==> Setup rate limit chain"
 ipxtables -t filter -N CLOUDRON_RATELIMIT || true

src/apps.js

@@ -4,8 +4,7 @@ exports = module.exports = {
     canAccess,
     isOperator,
     accessLevel,
-    removeInternalFields,
-    removeRestrictedFields,
+    pickFields,
 
     // database crud
     add,
@@ -135,6 +134,12 @@ exports = module.exports = {
     HEALTH_ERROR: 'error',
     HEALTH_DEAD: 'dead',
 
+    // app access levels
+    ACCESS_LEVEL_ADMIN: 'admin',
+    ACCESS_LEVEL_OPERATOR: 'operator',
+    ACCESS_LEVEL_USER: 'user',
+    ACCESS_LEVEL_NONE: '',
+
     // exported for testing
     _checkForPortBindingConflict: checkForPortBindingConflict,
     _validatePorts: validatePorts,
@@ -581,35 +586,35 @@ async function getStorageDir(app) {
     return path.join(volume.hostPath, app.storageVolumePrefix);
 }
 
-function removeCertificateKeys(app) {
-    if (app.certificate) delete app.certificate.key;
-    app.secondaryDomains.forEach(sd => { if (sd.certificate) delete sd.certificate.key; });
-    app.aliasDomains.forEach(ad => { if (ad.certificate) delete ad.certificate.key; });
-    app.redirectDomains.forEach(rd => { if (rd.certificate) delete rd.certificate.key; });
-}
+function pickFields(app, accessLevel) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof accessLevel, 'string');
 
-function removeInternalFields(app) {
-    const result = _.pick(app,
-        'id', 'appStoreId', 'installationState', 'error', 'runState', 'health', 'taskId',
-        'subdomain', 'domain', 'fqdn', 'certificate', 'crontab', 'upstreamUri',
-        'accessRestriction', 'manifest', 'portBindings', 'iconUrl', 'memoryLimit', 'cpuQuota', 'operators',
-        'sso', 'debugMode', 'reverseProxyConfig', 'enableBackup', 'creationTime', 'updateTime', 'ts', 'tags',
-        'label', 'notes', 'secondaryDomains', 'redirectDomains', 'aliasDomains', 'devices', 'env', 'enableAutomaticUpdate',
-        'storageVolumeId', 'storageVolumePrefix', 'mounts', 'enableTurn', 'enableRedis', 'checklist',
-        'enableMailbox', 'mailboxDisplayName', 'mailboxName', 'mailboxDomain', 'enableInbox', 'inboxName', 'inboxDomain');
+    if (accessLevel === exports.ACCESS_LEVEL_NONE) return null; // cannot happen!
 
-    removeCertificateKeys(result);
-    return result;
-}
+    let result;
+    if (accessLevel === exports.ACCESS_LEVEL_USER) {
+        result = _.pick(app,
+            'id', 'appStoreId', 'installationState', 'error', 'runState', 'health', 'taskId', 'accessRestriction',
+            'secondaryDomains', 'redirectDomains', 'aliasDomains', 'sso', 'subdomain', 'domain', 'fqdn', 'certificate',
+            'manifest', 'portBindings', 'iconUrl', 'creationTime', 'ts', 'tags', 'label', 'upstreamUri');
+    } else { // admin or operator
+        result = _.pick(app,
+            'id', 'appStoreId', 'installationState', 'error', 'runState', 'health', 'taskId',
+            'subdomain', 'domain', 'fqdn', 'certificate', 'crontab', 'upstreamUri',
+            'accessRestriction', 'manifest', 'portBindings', 'iconUrl', 'memoryLimit', 'cpuQuota', 'operators',
+            'sso', 'debugMode', 'reverseProxyConfig', 'enableBackup', 'creationTime', 'updateTime', 'ts', 'tags',
+            'label', 'notes', 'secondaryDomains', 'redirectDomains', 'aliasDomains', 'devices', 'env', 'enableAutomaticUpdate',
+            'storageVolumeId', 'storageVolumePrefix', 'mounts', 'enableTurn', 'enableRedis', 'checklist',
+            'enableMailbox', 'mailboxDisplayName', 'mailboxName', 'mailboxDomain', 'enableInbox', 'inboxName', 'inboxDomain');
+    }
 
-// non-admins can only see these
-function removeRestrictedFields(app) {
-    const result = _.pick(app,
-        'id', 'appStoreId', 'installationState', 'error', 'runState', 'health', 'taskId', 'accessRestriction', 'checklist',
-        'secondaryDomains', 'redirectDomains', 'aliasDomains', 'sso', 'subdomain', 'domain', 'fqdn', 'certificate',
-        'manifest', 'portBindings', 'iconUrl', 'creationTime', 'ts', 'tags', 'label', 'notes', 'enableBackup', 'upstreamUri');
+    // remove private certificate key
+    if (result.certificate) delete result.certificate.key;
+    result.secondaryDomains.forEach(sd => { if (sd.certificate) delete sd.certificate.key; });
+    result.aliasDomains.forEach(ad => { if (ad.certificate) delete ad.certificate.key; });
+    result.redirectDomains.forEach(rd => { if (rd.certificate) delete rd.certificate.key; });
 
-    removeCertificateKeys(result);
     return result;
 }
 
@@ -753,14 +758,6 @@ function postProcess(result) {
     delete result.errorJson;
 
     result.taskId = result.taskId ? String(result.taskId) : null;
-
-    // result.devices = {
-    //     '/dev/ttyUSB10': {
-    //         // future options
-    //     },
-    //     '/dev/hidraw0': {}
-    // };
-
     result.devices = result.devicesJson ? JSON.parse(result.devicesJson) : {};
     delete result.devicesJson;
 }
@@ -810,9 +807,9 @@ function canAccess(app, user) {
 }
 
 function accessLevel(app, user) {
-    if (isAdmin(user)) return 'admin';
-    if (isOperator(app, user)) return 'operator';
-    return canAccess(app, user) ? 'user' : null;
+    if (isAdmin(user)) return exports.ACCESS_LEVEL_ADMIN;
+    if (isOperator(app, user)) return exports.ACCESS_LEVEL_OPERATOR;
+    return canAccess(app, user) ? exports.ACCESS_LEVEL_USER : exports.ACCESS_LEVEL_NONE;
 }
 
 async function checkForPortBindingConflict(portBindings, options) {
@@ -2494,8 +2491,7 @@ async function unarchive(archive, data, auditSource) {
         domain = data.domain.toLowerCase(),
         overwriteDns = 'overwriteDns' in data ? data.overwriteDns : false;
 
-    const appConfig = backup.appConfig;
-    const { appStoreId, manifest } = appConfig;
+    const manifest = backup.manifest, appStoreId = backup.manifest.id;
 
     let error = validateSecondaryDomains(data.secondaryDomains || {}, manifest);
     if (error) throw error;
@@ -2517,7 +2513,8 @@ async function unarchive(archive, data, auditSource) {
 
     const appId = uuid.v4();
 
-    const dolly = _.pick(appConfig, 'memoryLimit', 'cpuQuota', 'crontab', 'reverseProxyConfig', 'env', 'servicesConfig',
+    // appConfig is null for pre-8.2 backups
+    const dolly = _.pick(backup.appConfig || {}, 'memoryLimit', 'cpuQuota', 'crontab', 'reverseProxyConfig', 'env', 'servicesConfig',
         'tags', 'label', 'enableMailbox', 'mailboxDisplayName', 'mailboxName', 'mailboxDomain', 'enableInbox', 'inboxName', 'inboxDomain', 'devices',
         'enableTurn', 'enableRedis', 'mounts', 'enableBackup', 'enableAutomaticUpdate', 'accessRestriction', 'operators', 'sso',
         'notes', 'checklist');
@@ -2529,7 +2526,8 @@ async function unarchive(archive, data, auditSource) {
         aliasDomains: [],
         mailboxDomain: data.domain, // archive's mailboxDomain may not exist
         runState: exports.RSTATE_RUNNING,
-        installationState: exports.ISTATE_PENDING_INSTALL
+        installationState: exports.ISTATE_PENDING_INSTALL,
+        sso: backup.appConfig ? backup.appConfig.sso : true // when no appConfig take a blind guess
     });
     obj.icon = (await archives.getIcons(archive.id))?.icon;
 

src/apptaskmanager.js

@@ -47,14 +47,14 @@ async function drain() {
 
         if (!fs.existsSync(path.dirname(logFile))) safe.fs.mkdirSync(path.dirname(logFile)); // ensure directory
 
-        scheduler.suspendJobs(appId);
+        scheduler.suspendAppJobs(appId);
 
         tasks.startTask(taskId, Object.assign(options, { logFile }), async function (error, result) {
             onFinished(error, result);
 
             delete gActiveTasks[appId];
             await locks.release(`${locks.TYPE_APP_PREFIX}${appId}`);
-            scheduler.resumeJobs(appId);
+            scheduler.resumeAppJobs(appId);
         });
     }
 

src/archives.js

@@ -17,7 +17,7 @@ const assert = require('assert'),
     safe = require('safetydance'),
     uuid = require('uuid');
 
-const ARCHIVE_FIELDS = [ 'archives.id', 'backupId', 'archives.creationTime', 'backups.remotePath', 'backups.appConfigJson', '(archives.icon IS NOT NULL) AS hasIcon', '(archives.appStoreIcon IS NOT NULL) AS hasAppStoreIcon' ];
+const ARCHIVE_FIELDS = [ 'archives.id', 'backupId', 'archives.creationTime', 'backups.remotePath', 'backups.manifestJson', 'backups.appConfigJson', '(archives.icon IS NOT NULL) AS hasIcon', '(archives.appStoreIcon IS NOT NULL) AS hasAppStoreIcon' ];
 
 function postProcess(result) {
     assert.strictEqual(typeof result, 'object');
@@ -25,6 +25,9 @@ function postProcess(result) {
     result.appConfig = result.appConfigJson ? safe.JSON.parse(result.appConfigJson) : null;
     delete result.appConfigJson;
 
+    result.manifest = result.manifestJson ? safe.JSON.parse(result.manifestJson) : null;
+    delete result.manifestJson;
+
     result.iconUrl = result.hasIcon || result.hasAppStoreIcon  ? `/api/v1/archives/${result.id}/icon` : null;
 
     return result;

src/backupformat/rsync.js

@@ -123,22 +123,20 @@ async function saveFsMetadata(dataLayout, metadataFile) {
         symlinks: []
     };
 
-    const MAX_FILES = 20000; // this is just a rough upper bound
-
     // we assume small number of files. spawnSync will raise a ENOBUFS error after maxBuffer
     for (const lp of dataLayout.localPaths()) {
-        const [emptyDirsError, emptyDirs] = await safe(shell.spawn('find', [lp, '-type', 'd', '-empty'], { encoding: 'utf8', maxLines: MAX_FILES }));
-        if (emptyDirsError && emptyDirsError.stdoutLineCount >= MAX_FILES) throw new BoxError(BoxError.FS_ERROR, `Too many empty directories. Run "find ${lp} -type d -empty" to investigate`);
+        const [emptyDirsError, emptyDirs] = await safe(shell.spawn('find', [lp, '-type', 'd', '-empty'], { encoding: 'utf8', maxLines: 50000 }));
+        if (emptyDirsError && emptyDirsError.stdoutLineCount >= 50000) throw new BoxError(BoxError.FS_ERROR, `Too many empty directories. Run "find ${lp} -type d -empty" to investigate`);
         if (emptyDirsError) throw emptyDirsError;
         if (emptyDirs.length) metadata.emptyDirs = metadata.emptyDirs.concat(emptyDirs.trim().split('\n').map((ed) => dataLayout.toRemotePath(ed)));
 
-        const [execFilesError, execFiles] = await safe(shell.spawn('find', [lp, '-type', 'f', '-executable'], { encoding: 'utf8', maxLines: MAX_FILES }));
-        if (execFilesError && execFilesError.stdoutLineCount >= MAX_FILES) throw new BoxError(BoxError.FS_ERROR, `Too many executable files. Run "find ${lp} -type f -executable" to investigate`);
+        const [execFilesError, execFiles] = await safe(shell.spawn('find', [lp, '-type', 'f', '-executable'], { encoding: 'utf8', maxLines: 20000 }));
+        if (execFilesError && execFilesError.stdoutLineCount >= 20000) throw new BoxError(BoxError.FS_ERROR, `Too many executable files. Run "find ${lp} -type f -executable" to investigate`);
         if (execFilesError) throw execFilesError;
         if (execFiles.length) metadata.execFiles = metadata.execFiles.concat(execFiles.trim().split('\n').map((ef) => dataLayout.toRemotePath(ef)));
 
-        const [symlinkFilesError, symlinkFiles] = await safe(shell.spawn('find', [lp, '-type', 'l'], { encoding: 'utf8', maxLines: MAX_FILES }));
-        if (symlinkFilesError && symlinkFilesError.stdoutLineCount >= MAX_FILES) throw new BoxError(BoxError.FS_ERROR, `Too many symlinks. Run "find ${lp} -type l" to investigate`);
+        const [symlinkFilesError, symlinkFiles] = await safe(shell.spawn('find', [lp, '-type', 'l'], { encoding: 'utf8', maxLines: 20000 }));
+        if (symlinkFilesError && symlinkFilesError.stdoutLineCount >= 20000) throw new BoxError(BoxError.FS_ERROR, `Too many symlinks. Run "find ${lp} -type l" to investigate`);
         if (symlinkFilesError) throw symlinkFilesError;
 
         if (symlinkFiles.length) metadata.symlinks = metadata.symlinks.concat(symlinkFiles.trim().split('\n').map((sl) => {

src/constants.js

@@ -42,6 +42,7 @@ exports = module.exports = {
     MYSQL_SERVICE_IPv4: '172.18.30.1',
     POSTGRESQL_SERVICE_IPv4: '172.18.30.2',
     MONGODB_SERVICE_IPv4: '172.18.30.3',
+    MAIL_SERVICE_IPv4: '172.18.30.4',
 
     NGINX_DEFAULT_CONFIG_FILE_NAME: 'default.conf',
 

src/cron.js

@@ -202,6 +202,7 @@ async function handleTimeZoneChanged(tz) {
 
     debug('handleTimeZoneChanged: recreating all jobs');
     await stopJobs();
+    await scheduler.deleteJobs(); // have to re-create with new tz
     await startJobs();
 }
 

src/docker.js

@@ -80,12 +80,12 @@ function parseImageRef(imageRef) {
     assert.strictEqual(typeof imageRef, 'string');
 
     // a ref is like registry.docker.com/cloudron/base:4.2.0@sha256:46da2fffb36353ef714f97ae8e962bd2c212ca091108d768ba473078319a47f4
-    // registry.docker.com is registry name . cloudron is namespace . base is image name . cloudron/base is repository path
+    // registry.docker.com is registry name . cloudron is (optional) namespace . base is image name . cloudron/base is repository path
     // registry.docker.com/cloudron/base is fullRepositoryName
     const result = { fullRepositoryName: null, registry: null, tag: null, digest: null };
     result.fullRepositoryName = imageRef.split(/[:@]/)[0];
     const parts = result.fullRepositoryName.split('/');
-    result.registry = parts.length === 3 ? parts[0] : null;
+    result.registry = parts[0].includes('.') ? parts[0] : null; // https://docs.docker.com/admin/faqs/general-faqs/#what-is-a-docker-id
     let remaining = imageRef.substr(result.fullRepositoryName.length);
     if (remaining.startsWith(':')) {
         result.tag = remaining.substr(1).split('@', 1)[0];

src/infra_version.js

@@ -13,7 +13,7 @@ exports = module.exports = {
     'images': {
         // 'base': 'registry.docker.com/cloudron/base:4.2.0@sha256:46da2fffb36353ef714f97ae8e962bd2c212ca091108d768ba473078319a47f4',
         'graphite': 'registry.docker.com/cloudron/graphite:3.4.3@sha256:75df420ece34b31a7ce8d45b932246b7f524c123e1854f5e8f115a9e94e33f20',
-        'mail': 'registry.docker.com/cloudron/mail:3.14.2@sha256:b760d8476194aff96050d48f856c283584fb7886ac3628a17c48811c22c8836d',
+        'mail': 'registry.docker.com/cloudron/mail:3.14.6@sha256:49ddb10b7355aa01e7b40c07ae9c583d171df9566491a40bb88ae91879f8f7eb',
         'mongodb': 'registry.docker.com/cloudron/mongodb:6.0.0@sha256:1108319805acfb66115aa96a8fdbf2cded28d46da0e04d171a87ec734b453d1e',
         'mysql': 'registry.docker.com/cloudron/mysql:3.4.3@sha256:8934c5ddcd69f24740d9a38f0de2937e47240238f3b8f5c482862eeccc5a21d2',
         'postgresql': 'registry.docker.com/cloudron/postgresql:5.3.1@sha256:eaea598aec086c90c0bb7bb8227bcde51b368bcca83d0082a4919bbb6f2d039f',

src/mail.js

@@ -397,6 +397,7 @@ async function checkDmarc(domain) {
     return dmarc;
 }
 
+// TODO: check ip6.arpa for IPv6 PTR
 async function checkPtr(mailFqdn) {
     assert.strictEqual(typeof mailFqdn, 'string');
 

src/mailserver.js

@@ -186,6 +186,7 @@ async function configureMail(mailFqdn, mailDomain, serviceConfig) {
                 --memory-swap -1 \
                 --dns 172.18.0.1 \
                 --dns-search=. \
+                --ip ${constants.MAIL_SERVICE_IPv4} \
                 -e CLOUDRON_MAIL_TOKEN=${cloudronToken} \
                 -e CLOUDRON_RELAY_TOKEN=${relayToken} \
                 -e LOGLEVEL=${logLevel} \
@@ -196,7 +197,7 @@ async function configureMail(mailFqdn, mailDomain, serviceConfig) {
                 ${readOnly} -v /run -v /tmp ${image} ${cmd}`;
 
     debug('configureMail: starting mail container');
-    await shell.bash(runCmd, {});
+    await shell.bash(runCmd, { encoding: 'utf8' });
 }
 
 async function restart() {

src/notifications.js

@@ -129,7 +129,7 @@ async function list(filters, page, perPage) {
 
     let query = `SELECT ${NOTIFICATION_FIELDS} FROM notifications`;
     if (where.length) query += ' WHERE ' + where.join(' AND ');
-    query += ' ORDER BY creationTime DESC LIMIT ?,?';
+    query += ' ORDER BY acknowledged ASC, creationTime DESC LIMIT ?,?';
 
     args.push((page-1)*perPage);
     args.push(perPage);

src/routes/apps.js

@@ -100,8 +100,9 @@ async function load(req, res, next) {
 function getApp(req, res, next) {
     assert.strictEqual(typeof req.app, 'object');
 
-    const result = apps.removeInternalFields(req.app);
-    result.accessLevel = apps.accessLevel(req.app, req.user);
+    const accessLevel = apps.accessLevel(req.app, req.user);
+    const result = apps.pickFields(req.app, accessLevel);
+    result.accessLevel = accessLevel;
 
     next(new HttpSuccess(200, result));
 }
@@ -109,13 +110,14 @@ function getApp(req, res, next) {
 async function listByUser(req, res, next) {
     assert.strictEqual(typeof req.user, 'object');
 
-    const [error, result] = await safe(apps.listByUser(req.user));
+    const [error, results] = await safe(apps.listByUser(req.user));
     if (error) return next(BoxError.toHttpError(error));
 
-    const filteredResult = result.map(r => {
-        const app = apps.removeRestrictedFields(r);
-        app.accessLevel = apps.accessLevel(r, req.user);
-        return app;
+    const filteredResult = results.map(app => {
+        const accessLevel = apps.accessLevel(app, req.user);
+        const result = apps.pickFields(app, accessLevel);
+        result.accessLevel = accessLevel;
+        return result;
     });
 
     next(new HttpSuccess(200, { apps: filteredResult }));

src/routes/test/archives-test.js

@@ -46,6 +46,7 @@ describe('Archives API', function () {
         expect(response.body.archives.length).to.be(1);
         expect(response.body.archives[0].id).to.be(archiveId);
         expect(response.body.archives[0].appConfig).to.eql(appBackup.appConfig);
+        expect(response.body.archives[0].manifest).to.eql(appBackup.manifest);
     });
 
     it('get valid archive', async function () {
@@ -53,6 +54,7 @@ describe('Archives API', function () {
             .query({ access_token: owner.token });
         expect(response.statusCode).to.equal(200);
         expect(response.body.appConfig).to.eql(appBackup.appConfig);
+        expect(response.body.manifest).to.eql(appBackup.manifest);
     });
 
     it('cannot get invalid archive', async function () {

src/scheduler.js

@@ -2,13 +2,16 @@
 
 exports = module.exports = {
     sync,
-    suspendJobs,
-    resumeJobs
+    deleteJobs,
+
+    suspendAppJobs,
+    resumeAppJobs
 };
 
 const apps = require('./apps.js'),
     assert = require('assert'),
     BoxError = require('./boxerror.js'),
+    cloudron = require('./cloudron.js'),
     constants = require('./constants.js'),
     { CronJob } = require('cron'),
     debug = require('debug')('box:scheduler'),
@@ -20,13 +23,13 @@ const gState = {}; // appId -> { containerId, schedulerConfig (manifest+crontab)
 const gSuspendedAppIds = new Set(); // suspended because some apptask is running
 
 // TODO: this should probably also stop existing jobs to completely prevent race but the code is not re-entrant
-function suspendJobs(appId) {
-    debug(`suspendJobs: ${appId}`);
+function suspendAppJobs(appId) {
+    debug(`suspendAppJobs: ${appId}`);
     gSuspendedAppIds.add(appId);
 }
 
-function resumeJobs(appId) {
-    debug(`resumeJobs: ${appId}`);
+function resumeAppJobs(appId) {
+    debug(`resumeAppJobs: ${appId}`);
     gSuspendedAppIds.delete(appId);
 }
 
@@ -60,6 +63,7 @@ async function createJobs(app, schedulerConfig) {
 
     const appId = app.id;
     const jobs = {};
+    const tz = await cloudron.getTimeZone();
 
     for (const taskName of Object.keys(schedulerConfig)) {
         const { schedule, command } = schedulerConfig[taskName];
@@ -87,7 +91,8 @@ async function createJobs(app, schedulerConfig) {
                 const [error] = await safe(runTask(appId, taskName)); //  put the app id in closure, so we don't use the outdated app object by mistake
                 if (error) debug(`could not run task ${taskName} : ${error.message}`);
             },
-            start: true
+            start: true,
+            timeZone: tz
         });
 
         jobs[taskName] = cronJob;
@@ -96,7 +101,7 @@ async function createJobs(app, schedulerConfig) {
     return jobs;
 }
 
-async function stopJobs(appId, appState) {
+async function deleteAppJobs(appId, appState) {
     assert.strictEqual(typeof appId, 'string');
     assert.strictEqual(typeof appState, 'object');
 
@@ -107,7 +112,16 @@ async function stopJobs(appId, appState) {
 
         const containerName = `${appId}-${taskName}`;
         const [error] = await safe(docker.deleteContainer(containerName));
-        if (error) debug(`stopJobs: failed to delete task container with name ${containerName} : ${error.message}`);
+        if (error) debug(`deleteAppJobs: failed to delete task container with name ${containerName} : ${error.message}`);
+    }
+}
+
+async function deleteJobs() {
+    for (const appId of Object.keys(gState)) {
+        debug(`deleteJobs: removing jobs of ${appId}`);
+        const [error] = await safe(deleteAppJobs(appId, gState[appId]));
+        if (error) debug(`deleteJobs: error stopping jobs of removed app ${appId}: ${error.message}`);
+        delete gState[appId];
     }
 }
 
@@ -122,7 +136,7 @@ async function sync() {
 
     for (const appId of removedAppIds) {
         debug(`sync: removing jobs of ${appId}`);
-        const [error] = await safe(stopJobs(appId, gState[appId]));
+        const [error] = await safe(deleteAppJobs(appId, gState[appId]));
         if (error) debug(`sync: error stopping jobs of removed app ${appId}: ${error.message}`);
         delete gState[appId];
     }
@@ -138,7 +152,7 @@ async function sync() {
 
         debug(`sync: clearing jobs of ${app.id} (${app.fqdn})`);
 
-        const [error] = await safe(stopJobs(app.id, appState));
+        const [error] = await safe(deleteAppJobs(app.id, appState));
         if (error) debug(`sync: error stopping jobs of ${app.id} : ${error.message}`);
 
         if (!schedulerConfig) { // updated app version removed scheduler addon

src/server.js

@@ -130,7 +130,7 @@ async function initializeExpressSync() {
     router.get ('/api/v1/eventlog/:eventId',                       token, authorizeAdmin, routes.eventlog.get);
 
     // updater
-    router.get ('/api/v1/updater/updates',                        token,  authorizeUser, routes.updater.getUpdateInfo);
+    router.get ('/api/v1/updater/updates',                        token,  authorizeUser, routes.updater.getUpdateInfo); // allowed for normal users to make it work for app operators
     router.post('/api/v1/updater/update',                   json, token, authorizeAdmin, routes.updater.update);
     router.post('/api/v1/updater/check_for_updates',        json, token, authorizeAdmin, routes.updater.checkForUpdates);
     router.get ('/api/v1/updater/autoupdate_pattern',             token, authorizeAdmin, routes.updater.getAutoupdatePattern);

src/services.js

@@ -1035,7 +1035,7 @@ async function startTurn(existingInfra) {
     await docker.deleteContainer('turn');
 
     debug('startTurn: starting turn container');
-    await shell.bash(runCmd, {});
+    await shell.bash(runCmd, { encoding: 'utf8' });
 
     if (existingInfra.version !== 'none' && existingInfra.images.turn !== image) await docker.deleteImage(existingInfra.images.turn);
 }
@@ -1245,7 +1245,7 @@ async function startMysql(existingInfra) {
     await docker.deleteContainer('mysql');
 
     debug('startMysql: starting mysql container');
-    await shell.bash(runCmd, {});
+    await shell.bash(runCmd, { encoding: 'utf8' });
 
     if (!serviceConfig.recoveryMode) {
         await waitForContainer('mysql', 'CLOUDRON_MYSQL_TOKEN');
@@ -1465,7 +1465,7 @@ async function startPostgresql(existingInfra) {
     await docker.deleteContainer('postgresql');
 
     debug('startPostgresql: starting postgresql container');
-    await shell.bash(runCmd, {});
+    await shell.bash(runCmd, { encoding: 'utf8' });
 
     if (!serviceConfig.recoveryMode) {
         await waitForContainer('postgresql', 'CLOUDRON_POSTGRESQL_TOKEN');
@@ -1616,7 +1616,7 @@ async function startMongodb(existingInfra) {
     }
 
     debug('startMongodb: starting mongodb container');
-    await shell.bash(runCmd, {});
+    await shell.bash(runCmd, { encoding: 'utf8' });
 
     if (!serviceConfig.recoveryMode) {
         await waitForContainer('mongodb', 'CLOUDRON_MONGODB_TOKEN');
@@ -1789,7 +1789,7 @@ async function startGraphite(existingInfra) {
     if (upgrading) await shell.promises.sudo([ RMADDONDIR_CMD, 'graphite' ], {});
 
     debug('startGraphite: starting graphite container');
-    await shell.bash(runCmd, {});
+    await shell.bash(runCmd, { encoding: 'utf8' });
 
     if (existingInfra.version !== 'none' && existingInfra.images.graphite !== image) await docker.deleteImage(existingInfra.images.graphite);
 
@@ -1938,7 +1938,7 @@ async function setupRedis(app, options) {
 
     const [inspectError, result] = await safe(docker.inspect(redisName));
     if (inspectError) {
-        await shell.bash(runCmd, {});
+        await shell.bash(runCmd, { encoding: 'utf8' });
     } else { // fast path
         debug(`Re-using existing redis container with state: ${JSON.stringify(result.State)}`);
     }

src/sftp.js

@@ -124,7 +124,7 @@ async function start(existingInfra) {
     await docker.deleteContainer('sftp');
 
     debug('startSftp: starting sftp container');
-    await shell.bash(runCmd, {});
+    await shell.bash(runCmd, { encoding: 'utf8' });
 
     if (existingInfra.version !== 'none' && existingInfra.images.sftp !== image) await docker.deleteImage(existingInfra.images.sftp);
 }

src/test/apps-test.js

@@ -11,7 +11,8 @@ const apps = require('../apps.js'),
     common = require('./common.js'),
     expect = require('expect.js'),
     Location = require('../location.js'),
-    safe = require('safetydance');
+    safe = require('safetydance'),
+    users = require('../users.js');
 
 describe('Apps', function () {
     const { domainSetup, cleanup, app, admin, user , domain } = common;
@@ -158,8 +159,8 @@ describe('Apps', function () {
     });
 
     describe('canAccess', function () {
-        const someuser = { id: 'someuser', groupIds: [], role: 'user' };
-        const adminuser = { id: 'adminuser', groupIds: [ 'groupie' ], role: 'admin' };
+        const someuser = { id: 'someuser', groupIds: [], role: users.ROLE_USER };
+        const adminuser = { id: 'adminuser', groupIds: [ 'groupie' ], role: users.ROLE_ADMIN };
 
         it('returns true for unrestricted access', function () {
             expect(apps.canAccess({ accessRestriction: null }, someuser)).to.be(true);
@@ -196,7 +197,7 @@ describe('Apps', function () {
 
     describe('isOperator', function () {
         const someuser = { id: 'someuser', groupIds: [], role: 'user' };
-        const adminuser = { id: 'adminuser', groupIds: [ 'groupie' ], role: 'admin' };
+        const adminuser = { id: 'adminuser', groupIds: [ 'groupie' ], role: users.ROLE_ADMIN };
 
         it('returns false for unrestricted access', function () {
             expect(apps.isOperator({ operators: null }, someuser)).to.be(false);
@@ -232,22 +233,22 @@ describe('Apps', function () {
     });
 
     describe('accessLevel', function () {
-        const someuser = { id: 'someuser', groupIds: [ 'ops' ], role: 'user' };
-        const adminuser = { id: 'adminuser', groupIds: [ 'groupie' ], role: 'admin' };
+        const someuser = { id: 'someuser', groupIds: [ 'ops' ], role: users.ROLE_USER };
+        const adminuser = { id: 'adminuser', groupIds: [ 'groupie' ], role: users.ROLE_ADMIN };
 
         it('return user for normal user', function () {
-            expect(apps.accessLevel({ accessRestriction: null, operators: null }, someuser)).to.be('user');
-            expect(apps.accessLevel({ accessRestriction: null, operators: { users: [ ], groups: [ 'groupie' ] } }, someuser)).to.be('user');
+            expect(apps.accessLevel({ accessRestriction: null, operators: null }, someuser)).to.be(apps.ACCESS_LEVEL_USER);
+            expect(apps.accessLevel({ accessRestriction: null, operators: { users: [ ], groups: [ 'groupie' ] } }, someuser)).to.be(apps.ACCESS_LEVEL_USER);
         });
 
         it('returns operator for operator user', function () {
-            expect(apps.accessLevel({ accessRestriction: null, operators: { users: [ 'someuser' ], groups: [ 'groupie' ] } }, someuser)).to.be('operator');
-            expect(apps.accessLevel({ accessRestriction: null, operators: { users: [], groups: [ 'ops' ] } }, someuser)).to.be('operator');
+            expect(apps.accessLevel({ accessRestriction: null, operators: { users: [ 'someuser' ], groups: [ 'groupie' ] } }, someuser)).to.be(apps.ACCESS_LEVEL_OPERATOR);
+            expect(apps.accessLevel({ accessRestriction: null, operators: { users: [], groups: [ 'ops' ] } }, someuser)).to.be(apps.ACCESS_LEVEL_OPERATOR);
         });
 
         it('returns admin for admin user', function () {
-            expect(apps.accessLevel({ accessRestriction: null, operators: null }, adminuser)).to.be('admin');
-            expect(apps.accessLevel({ accessRestriction: null, operators: { users: [], groups: [] } }, adminuser)).to.be('admin');
+            expect(apps.accessLevel({ accessRestriction: null, operators: null }, adminuser)).to.be(apps.ACCESS_LEVEL_ADMIN);
+            expect(apps.accessLevel({ accessRestriction: null, operators: { users: [], groups: [] } }, adminuser)).to.be(apps.ACCESS_LEVEL_ADMIN);
         });
     });
 

src/test/branding-test.js

@@ -49,6 +49,6 @@ describe('Branding', function () {
 
     it('can render footer with YEAR', async function () {
         await branding.setFooter('BigFoot Inc %YEAR%', auditSource);
-        expect(await branding.renderFooter()).to.be('BigFoot Inc 2024');
+        expect(await branding.renderFooter()).to.be('BigFoot Inc 2025');
     });
 });

src/test/docker-test.js

@@ -27,5 +27,6 @@ describe('docker', function () {
         expect(docker.parseImageRef('cloudron/base@sha256:xx')).to.eql({ fullRepositoryName: 'cloudron/base', registry: null, tag: null, digest: 'xx' });
         expect(docker.parseImageRef('cloudron/base:4.2.0@sha256:xx')).to.eql({ fullRepositoryName: 'cloudron/base', registry: null, tag: '4.2.0', digest: 'xx' });
         expect(docker.parseImageRef('registry.com/cloudron/base:4.2.0@sha256:xx')).to.eql({ fullRepositoryName: 'registry.com/cloudron/base', registry: 'registry.com', tag: '4.2.0', digest: 'xx' });
+        expect(docker.parseImageRef('registry.com/base:4.2.0@sha256:xx')).to.eql({ fullRepositoryName: 'registry.com/base', registry: 'registry.com', tag: '4.2.0', digest: 'xx' }); // optional namespace
     });
 });

src/test/dockerproxy-test.js

@@ -28,6 +28,8 @@ async function exec(cmd) {
     });
 }
 
+const BASE_IMAGE = 'registry.docker.com/cloudron/base:4.2.0@sha256:46da2fffb36353ef714f97ae8e962bd2c212ca091108d768ba473078319a47f4';
+
 describe('Dockerproxy', function () {
     let containerId;
     const { setup, cleanup } = common;
@@ -38,7 +40,7 @@ describe('Dockerproxy', function () {
         await syslogServer.start();
         await dockerProxy.start();
 
-        const stdout = await exec(`${DOCKER} run -d ${infra.images.base} "bin/bash" "-c" "while true; do echo 'perpetual walrus'; sleep 1; done"`);
+        const stdout = await exec(`${DOCKER} run -d ${BASE_IMAGE} "bin/bash" "-c" "while true; do echo 'perpetual walrus'; sleep 1; done"`);
         containerId = stdout.slice(0, -1); // removes the trailing \n
     });
 
@@ -59,13 +61,13 @@ describe('Dockerproxy', function () {
     });
 
     it('can create container', async function () {
-        const cmd = `${DOCKER} run ${infra.images.base} "/bin/bash" "-c" "echo 'hello'"`;
+        const cmd = `${DOCKER} run ${BASE_IMAGE} "/bin/bash" "-c" "echo 'hello'"`;
         const stdout = await exec(cmd);
         expect(stdout).to.contain('hello');
     });
 
     it('proxy overwrites the container network option', async function () {
-        const cmd = `${DOCKER} run --network ifnotrewritethiswouldfail ${infra.images.base} "/bin/bash" "-c" "echo 'hello'"`;
+        const cmd = `${DOCKER} run --network ifnotrewritethiswouldfail ${BASE_IMAGE} "/bin/bash" "-c" "echo 'hello'"`;
         const stdout = await exec(cmd);
         expect(stdout).to.contain('hello');
     });

src/test/volumes-test.js

@@ -9,6 +9,7 @@
 const BoxError = require('../boxerror.js'),
     common = require('./common.js'),
     expect = require('expect.js'),
+    paths = require('../paths.js'),
     safe = require('safetydance'),
     volumes = require('../volumes.js');
 
@@ -73,4 +74,26 @@ describe('Volumes', function () {
     it('can del volume', async function () {
         await volumes.del(volume, auditSource);
     });
+
+    const badPaths = [
+        '/opt/data/../bar', // not normalized
+        'opt/data/bar', // not absolute
+        '/', // root
+        '/mnt', // top level itself is reserved
+        '/usr/bin', // reserved
+        paths.VOLUMES_MOUNT_DIR, // reserved
+        paths.VOLUMES_MOUNT_DIR + '/', // also reserved
+        paths.VOLUMES_MOUNT_DIR + '/something', // also reserved
+        '/media/cloudron-test-music/root', // realpath won't match
+        '/media/cloudron-test-music/root/', // realpath won't match
+        '/media/cloudron-test-music/file', // need directory
+        '/media/cloudron-test-music/randompath', // need directory
+    ];
+
+    for (const p of badPaths) {
+        it(`validateHostPath - ${p}`, function () {
+            const error = volumes._validateHostPath(p);
+            expect(error.reason).to.be(BoxError.BAD_FIELD);
+        });
+    }
 });

src/volumes.js

@@ -10,7 +10,10 @@ exports = module.exports = {
     getStatus,
     removePrivateFields,
 
-    mountAll
+    mountAll,
+
+    // exported for testing
+    _validateHostPath: validateHostPath
 };
 
 const assert = require('assert'),
@@ -45,28 +48,25 @@ function validateName(name) {
     return null;
 }
 
-function validateHostPath(hostPath, mountType) {
+function validateHostPath(hostPath) {
     assert.strictEqual(typeof hostPath, 'string');
-    assert.strictEqual(typeof mountType, 'string');
 
     if (path.normalize(hostPath) !== hostPath) return new BoxError(BoxError.BAD_FIELD, 'hostPath must contain a normalized path');
     if (!path.isAbsolute(hostPath)) return new BoxError(BoxError.BAD_FIELD, 'hostPath must be an absolute path');
+    // otherwise, we could follow some symlink to mount paths outside the allowed paths
+    if (safe.fs.realpathSync(hostPath) !== hostPath) return new BoxError(BoxError.BAD_FIELD, 'hostPath must be a realpath without symlinks');
 
     if (hostPath === '/') return new BoxError(BoxError.BAD_FIELD, 'hostPath cannot be /');
 
-    if (!hostPath.endsWith('/')) hostPath = hostPath + '/'; // ensure trailing slash for the prefix matching to work
+    const allowedPaths = [ '/mnt', '/media', '/srv', '/opt' ];
+    if (!allowedPaths.some(p => hostPath.startsWith(p + '/'))) return new BoxError(BoxError.BAD_FIELD, 'hostPath must be under /mnt, /media, /opt or /srv');
 
-    const allowedPaths = [ '/mnt/', '/media/', '/srv/', '/opt/' ];
-    if (!allowedPaths.some(p => hostPath.startsWith(p))) return new BoxError(BoxError.BAD_FIELD, 'hostPath must be under /mnt, /media, /opt or /srv');
+    const reservedPaths = [ `${paths.VOLUMES_MOUNT_DIR}` ];
+    if (reservedPaths.some(p => hostPath === p || hostPath.startsWith(p + '/'))) return new BoxError(BoxError.BAD_FIELD, 'hostPath is reserved');
 
-    const reservedPaths = [ `${paths.VOLUMES_MOUNT_DIR}/` ];
-    if (reservedPaths.some(p => hostPath.startsWith(p))) return new BoxError(BoxError.BAD_FIELD, 'hostPath is reserved');
-
-    if (!constants.TEST) { // we expect user to have already mounted this
-        const stat = safe.fs.lstatSync(hostPath);
-        if (!stat) return new BoxError(BoxError.BAD_FIELD, 'hostPath does not exist. Please create it on the server first');
-        if (!stat.isDirectory()) return new BoxError(BoxError.BAD_FIELD, 'hostPath is not a directory');
-    }
+    const stat = safe.fs.lstatSync(hostPath);
+    if (!stat) return new BoxError(BoxError.BAD_FIELD, 'hostPath does not exist. Please create it on the server first');
+    if (!stat.isDirectory()) return new BoxError(BoxError.BAD_FIELD, 'hostPath is not a directory');
 
     return null;
 }
@@ -87,7 +87,7 @@ async function add(volume, auditSource) {
 
     let hostPath;
     if (mountType === 'mountpoint' || mountType === 'filesystem') {
-        error = validateHostPath(mountOptions.hostPath, mountType);
+        error = validateHostPath(mountOptions.hostPath);
         if (error) throw error;
         hostPath = mountOptions.hostPath;
     } else {