cloudron-support: suggest removing nodejs apt

needs a lock because the cert code also restart mail server from tasks

CHANGES

@@ -2864,3 +2864,21 @@
 * fix "happy eyeballs" quirk in nodejs
 * Update nodejs to 20.18.0
 
+[8.2.0]
+* rsync: show better error message with too many empty dirs, symlinks or executables
+* mail: update Solr to 8.11.4
+* mail: update Haraka to 3.0.5
+* Add sqlite3 addon
+* docker: update docker to 27.3.1
+* du: add exclude file to skip filesystem usage checks
+* mail: attachment search
+* oidc: use cloudron name as provider name
+* groups: add eventlog
+* resources: allow mounting devices into apps
+* remove global lock
+* hetzner: add helsinki object storage location
+* backups: implement app archive
+* notifications: per user email notification config
+* postgres: enable vector extension
+* docker: fallback to downloading images from quay if dockerhub does not work
+

dashboard/develop.sh

@@ -3,7 +3,7 @@
 set -eu
 
 echo "=> Set API origin"
-export VITE_API_ORIGIN="https://my.nebulon.space"
+export VITE_API_ORIGIN="${DASHBOARD_DEVELOPMENT_ORIGIN}"
 
 # only really used for prod builds to bust cache
 export VITE_CACHE_ID="develop"

dashboard/index.html

@@ -85,7 +85,7 @@
   <script type="text/javascript" src="/views/settings.js?%VITE_CACHE_ID%"></script>
   <script type="text/javascript" src="/views/support.js?%VITE_CACHE_ID%"></script>
   <script type="text/javascript" src="/views/system.js?%VITE_CACHE_ID%"></script>
-  <script type="text/javascript" src="/views/user-settings.js?%VITE_CACHE_ID%"></script>
+  <script type="text/javascript" src="/views/user-directory.js?%VITE_CACHE_ID%"></script>
   <script type="text/javascript" src="/views/users.js?%VITE_CACHE_ID%"></script>
   <script type="text/javascript" src="/views/volumes.js?%VITE_CACHE_ID%"></script>
 
@@ -184,7 +184,7 @@
                 <li ng-show="user.isAtLeastAdmin"><a href="#/network" ng-click="closeNavbar()"><i class="fas fa-network-wired fa-fw"></i> {{ 'network.title' | tr }}</a></li>
                 <li ng-show="user.isAtLeastAdmin"><a href="#/services" ng-click="closeNavbar()"><i class="fa fa-cogs fa-fw"></i> {{ 'services.title' | tr }}</a></li>
                 <li ng-show="user.isAtLeastAdmin"><a href="#/settings" ng-click="closeNavbar()"><i class="fa fa-wrench fa-fw"></i> {{ 'settings.title' | tr }}</a></li>
-                <li ng-show="user.isAtLeastAdmin"><a href="#/usersettings" ng-click="closeNavbar()"><i class="fa fa-users-gear fa-fw"></i> {{ 'users.title' | tr }}</a></li>
+                <li ng-show="user.isAtLeastAdmin"><a href="#/user-directory" ng-click="closeNavbar()"><i class="fa fa-users-gear fa-fw"></i> {{ 'users.title' | tr }}</a></li>
                 <li ng-show="user.isAtLeastAdmin"><a href="#/volumes" ng-click="closeNavbar()"><i class="fa fa-hdd fa-fw"></i> {{ 'volumes.title' | tr }}</a></li>
                 <li ng-show="user.isAtLeastAdmin" class="divider"></li>
                 <li ng-show="user.isAtLeastOwner"><a href="#/support" ng-click="closeNavbar()"><i class="fa fa-comment fa-fw"></i> {{ 'support.title' | tr }}</a></li>

dashboard/package.json

@@ -7,27 +7,27 @@
   },
   "type": "module",
   "dependencies": {
-    "@eslint/js": "^9.14.0",
+    "@eslint/js": "^9.16.0",
     "@fontsource/noto-sans": "^5.1.0",
-    "@fortawesome/fontawesome-free": "^6.6.0",
-    "@vitejs/plugin-vue": "^5.1.4",
+    "@fortawesome/fontawesome-free": "^6.7.1",
+    "@vitejs/plugin-vue": "^5.2.1",
     "@xterm/addon-attach": "^0.11.0",
     "@xterm/addon-fit": "^0.10.0",
     "@xterm/xterm": "^5.5.0",
     "anser": "^2.3.0",
     "bootstrap-sass": "^3.4.3",
-    "chart.js": "^4.4.6",
-    "eslint-plugin-vue": "^9.30.0",
+    "chart.js": "^4.4.7",
+    "eslint-plugin-vue": "^9.32.0",
     "filesize": "^10.1.6",
     "jquery": "^3.7.1",
-    "marked": "^14.1.4",
+    "marked": "^15.0.3",
     "moment": "^2.30.1",
-    "pankow": "^2.3.4",
-    "pankow-viewers": "^1.0.9",
-    "sass": "^1.80.6",
-    "vite": "^5.4.10",
-    "vue": "^3.5.12",
-    "vue-i18n": "^10.0.4",
-    "vue-router": "^4.4.5"
+    "pankow": "^2.4.2",
+    "pankow-viewers": "^1.0.11",
+    "sass": "^1.82.0",
+    "vite": "^6.0.3",
+    "vue": "^3.5.13",
+    "vue-i18n": "^10.0.5",
+    "vue-router": "^4.5.0"
   }
 }

dashboard/public/js/client.js

@@ -4,20 +4,20 @@
 
 // keep in sync with box/src/notfications.js
 const NOTIFICATION_TYPES = {
-    ALERT_CLOUDRON_INSTALLED: 'cloudronInstalled',
-    ALERT_CLOUDRON_UPDATED: 'cloudronUpdated',
-    ALERT_CLOUDRON_UPDATE_FAILED: 'cloudronUpdateFailed',
-    ALERT_CERTIFICATE_RENEWAL_FAILED: 'certificateRenewalFailed',
-    ALERT_BACKUP_CONFIG: 'backupConfig',
-    ALERT_DISK_SPACE: 'diskSpace',
-    ALERT_MAIL_STATUS: 'mailStatus',
-    ALERT_REBOOT: 'reboot',
-    ALERT_BOX_UPDATE: 'boxUpdate',
-    ALERT_UPDATE_UBUNTU: 'ubuntuUpdate',
-    ALERT_MANUAL_APP_UPDATE: 'manualAppUpdate',
-    ALERT_APP_OOM: 'appOutOfMemory',
-    ALERT_APP_UPDATED: 'appUpdated',
-    ALERT_BACKUP_FAILED: 'backupFailed',
+    CLOUDRON_INSTALLED: 'cloudronInstalled',
+    CLOUDRON_UPDATED: 'cloudronUpdated',
+    CLOUDRON_UPDATE_FAILED: 'cloudronUpdateFailed',
+    CERTIFICATE_RENEWAL_FAILED: 'certificateRenewalFailed',
+    BACKUP_CONFIG: 'backupConfig',
+    DISK_SPACE: 'diskSpace',
+    MAIL_STATUS: 'mailStatus',
+    REBOOT: 'reboot',
+    BOX_UPDATE: 'boxUpdate',
+    UPDATE_UBUNTU: 'ubuntuUpdate',
+    MANUAL_APP_UPDATE: 'manualAppUpdate',
+    APP_OOM: 'appOutOfMemory',
+    APP_UPDATED: 'appUpdated',
+    BACKUP_FAILED: 'backupFailed',
 };
 
 // keep in sync with box/src/apps.js
@@ -166,6 +166,7 @@ const REGIONS_WASABI = [
 
 const REGIONS_HETZNER = [
     { name: 'Falkenstein (FSN1)', value: 'https://fsn1.your-objectstorage.com' },
+    { name: 'Helsinki (HEL1)', value: 'https://hel1.your-objectstorage.com' },
     { name: 'Nuremberg (NBG1)', value: 'https://nbg1.your-objectstorage.com' }
 ];
 
@@ -680,7 +681,8 @@ angular.module('Application').service('Client', ['$http', '$interval', '$timeout
             source: null,
             avatarUrl: null,
             avatarType: null,
-            hasBackgroundImage: false
+            hasBackgroundImage: false,
+            notificationConfig: []
         };
         this._config = {
             consoleServerOrigin: null,
@@ -812,6 +814,7 @@ angular.module('Application').service('Client', ['$http', '$interval', '$timeout
         this._userInfo.avatarUrl = userInfo.avatarUrl + '?ts=' + Date.now(); // we add the timestamp to avoid caching
         this._userInfo.avatarType = userInfo.avatarType;
         this._userInfo.hasBackgroundImage = userInfo.hasBackgroundImage;
+        this._userInfo.notificationConfig = userInfo.notificationConfig;
         this._userInfo.isAtLeastOwner = [ ROLES.OWNER ].indexOf(userInfo.role) !== -1;
         this._userInfo.isAtLeastAdmin = [ ROLES.OWNER, ROLES.ADMIN ].indexOf(userInfo.role) !== -1;
         this._userInfo.isAtLeastMailManager = [ ROLES.OWNER, ROLES.ADMIN, ROLES.MAIL_MANAGER ].indexOf(userInfo.role) !== -1;
@@ -940,7 +943,7 @@ angular.module('Application').service('Client', ['$http', '$interval', '$timeout
         });
     };
 
-    Client.prototype.installApp = function (id, manifest, title, config, callback) {
+    Client.prototype.installApp = function (id, manifest, config, callback) {
         var data = {
             appStoreId: id + '@' + manifest.version,
             subdomain: config.subdomain,
@@ -952,10 +955,11 @@ angular.module('Application').service('Client', ['$http', '$interval', '$timeout
             key: config.key,
             sso: config.sso,
             overwriteDns: config.overwriteDns,
-            upstreamUri: config.upstreamUri
+            upstreamUri: config.upstreamUri,
+            backupId: config.backupId // when restoring from archive
         };
 
-        post('/api/v1/apps/install', data, null, function (error, data, status) {
+        post('/api/v1/apps', data, null, function (error, data, status) {
             if (error) return callback(error);
             if (status !== 202) return callback(new ClientError(status, data));
 
@@ -1003,6 +1007,17 @@ angular.module('Application').service('Client', ['$http', '$interval', '$timeout
         });
     };
 
+    Client.prototype.archiveApp = function (appId, backupId, callback) {
+        var data = { backupId: backupId };
+
+        post('/api/v1/apps/' + appId + '/archive', data, null, function (error, data, status) {
+            if (error) return callback(error);
+            if (status !== 202) return callback(new ClientError(status, data));
+
+            callback(null);
+        });
+    };
+
     Client.prototype.uninstallApp = function (appId, callback) {
         var data = {};
 
@@ -1486,6 +1501,41 @@ angular.module('Application').service('Client', ['$http', '$interval', '$timeout
         });
     };
 
+    Client.prototype.listArchives = function (callback) {
+        var config = {
+            params: {
+                page: 1,
+                per_page: 100
+            }
+        };
+
+        get('/api/v1/archives', config, function (error, data, status) {
+            if (error) return callback(error);
+            if (status !== 200) return callback(new ClientError(status, data));
+
+            callback(null, data.archives);
+        });
+    };
+
+    Client.prototype.deleteArchive = function (id, callback) {
+        del('/api/v1/archives/' + id, null, function (error, data, status) {
+            if (error) return callback(error);
+            if (status !== 204) return callback(new ClientError(status, data));
+
+            callback(null);
+        });
+    };
+
+    Client.prototype.unarchiveApp = function (archiveId, data, callback) {
+        post('/api/v1/archives/' + archiveId + '/unarchive', data, null, function (error, data, status) {
+            if (error) return callback(error);
+            if (status !== 202) return callback(new ClientError(status, data));
+
+            callback(null, data);
+        });
+    };
+
+
     Client.prototype.getBackups = function (callback) {
         var page = 1;
         var perPage = 100;
@@ -2314,7 +2364,7 @@ angular.module('Application').service('Client', ['$http', '$interval', '$timeout
     Client.prototype.updateApplink = function (id, data, callback) {
         post('/api/v1/applinks/' + id, data, null, function (error, data, status) {
             if (error) return callback(error);
-            if (status !== 202) return callback(new ClientError(status, data));
+            if (status !== 200) return callback(new ClientError(status, data));
 
             callback(null);
         });
@@ -2409,6 +2459,15 @@ angular.module('Application').service('Client', ['$http', '$interval', '$timeout
         });
     };
 
+    Client.prototype.setNotificationConfig = function (notificationConfig, callback) {
+        post('/api/v1/profile/notification_config', { notificationConfig }, null, function (error, data, status) {
+            if (error) return callback(error);
+            if (status !== 204) return callback(new ClientError(status, data));
+
+            callback(null, data);
+        });
+    };
+
     Client.prototype.setProfileEmail = function (email, password, callback) {
         post('/api/v1/profile/email', { email: email, password: password }, null, function (error, data, status) {
             if (error) return callback(error);
@@ -3106,18 +3165,18 @@ angular.module('Application').service('Client', ['$http', '$interval', '$timeout
         });
     };
 
-    Client.prototype.getSolrConfig = function (callback) {
+    Client.prototype.getFtsConfig = function (callback) {
         var config = {};
 
-        get('/api/v1/mailserver/solr_config', config, function (error, data, status) {
+        get('/api/v1/mailserver/fts_config', config, function (error, data, status) {
             if (error) return callback(error);
             if (status !== 200) return callback(new ClientError(status, data));
             callback(null, data);
         });
     };
 
-    Client.prototype.setSolrConfig = function (enabled, callback) {
-        post('/api/v1/mailserver/solr_config', { enabled: enabled }, null, function (error, data, status) {
+    Client.prototype.setFtsConfig = function (state, callback) {
+        post('/api/v1/mailserver/fts_config', { enable: state }, null, function (error, data, status) {
             if (error) return callback(error);
             if (status !== 200) return callback(new ClientError(status, data));
 
@@ -3666,10 +3725,18 @@ angular.module('Application').service('Client', ['$http', '$interval', '$timeout
         var ACTION_APP_STOP = 'app.stop';
         var ACTION_APP_RESTART = 'app.restart';
 
+        var ACTION_ARCHIVES_ADD = 'archives.add';
+        var ACTION_ARCHIVES_DEL = 'archives.del';
+
         var ACTION_BACKUP_FINISH = 'backup.finish';
         var ACTION_BACKUP_START = 'backup.start';
         var ACTION_BACKUP_CLEANUP_START = 'backup.cleanup.start';
         var ACTION_BACKUP_CLEANUP_FINISH = 'backup.cleanup.finish';
+
+        var ACTION_BRANDING_AVATAR = 'branding.avatar';
+        var ACTION_BRANDING_NAME = 'branding.name';
+        var ACTION_BRANDING_FOOTER = 'branding.footer';
+
         var ACTION_CERTIFICATE_NEW = 'certificate.new';
         var ACTION_CERTIFICATE_RENEWAL = 'certificate.renew';
         var ACTION_CERTIFICATE_CLEANUP = 'certificate.cleanup';
@@ -3684,6 +3751,11 @@ angular.module('Application').service('Client', ['$http', '$interval', '$timeout
 
         var ACTION_EXTERNAL_LDAP_CONFIGURE = 'externalldap.configure';
 
+        var ACTION_GROUP_ADD = 'group.add';
+        var ACTION_GROUP_UPDATE = 'group.update';
+        var ACTION_GROUP_REMOVE = 'group.remove';
+        var ACTION_GROUP_MEMBERSHIP = 'group.membership';
+
         var ACTION_INSTALL_FINISH = 'cloudron.install.finish';
 
         var ACTION_START = 'cloudron.start';
@@ -3911,6 +3983,12 @@ angular.module('Application').service('Client', ['$http', '$interval', '$timeout
             if (!data.app) return '';
             return appName('', data.app, 'App') + ' was restarted';
 
+        case ACTION_ARCHIVES_ADD:
+            return 'Backup ' + data.backupId + ' added to archive';
+
+        case ACTION_ARCHIVES_DEL:
+            return 'Backup ' + data.backupId + ' deleted from archive';
+
         case ACTION_BACKUP_START:
             return 'Backup started';
 
@@ -3927,6 +4005,15 @@ angular.module('Application').service('Client', ['$http', '$interval', '$timeout
         case ACTION_BACKUP_CLEANUP_FINISH:
             return data.errorMessage ? 'Backup cleaner errored: ' + data.errorMessage : 'Backup cleaner removed ' + (data.removedBoxBackupPaths ? data.removedBoxBackupPaths.length : '0') + ' backups';
 
+        case ACTION_BRANDING_AVATAR:
+            return 'Cloudron Avatar Changed';
+
+        case ACTION_BRANDING_NAME:
+            return 'Cloudron Name set to ' + data.name;
+
+        case ACTION_BRANDING_FOOTER:
+            return 'Cloudron Footer set to ' + data.footer;
+
         case ACTION_CERTIFICATE_NEW:
             return 'Certificate install for ' + data.domain + (errorMessage ? ' failed' : ' succeeded');
 
@@ -3962,6 +4049,18 @@ angular.module('Application').service('Client', ['$http', '$interval', '$timeout
                 return 'External Directory set to ' + data.config.url + ' (' + data.config.provider + ')';
             }
 
+        case ACTION_GROUP_ADD:
+            return 'Group ' + data.name + ' was added';
+
+        case ACTION_GROUP_UPDATE:
+            return 'Group name changed from ' + data.oldName + ' to ' + data.group.name;
+
+        case ACTION_GROUP_REMOVE:
+            return 'Group ' + data.group.name + ' was removed';
+
+        case ACTION_GROUP_MEMBERSHIP:
+            return 'Group membership of ' + data.group.name + ' changed. Now was ' + data.userIds.length + ' member(s).';
+
         case ACTION_INSTALL_FINISH:
             return 'Cloudron version ' + data.version + ' installed';
 

dashboard/public/js/index.js

@@ -49,9 +49,9 @@ app.config(['$routeProvider', function ($routeProvider) {
     }).when('/users', {
         controller: 'UsersController',
         templateUrl: 'views/users.html?<%= revision %>'
-    }).when('/usersettings', {
+    }).when('/user-directory', {
         controller: 'UserSettingsController',
-        templateUrl: 'views/user-settings.html?<%= revision %>'
+        templateUrl: 'views/user-directory.html?<%= revision %>'
     }).when('/app/:appId/:view?', {
         controller: 'AppController',
         templateUrl: 'views/app.html?<%= revision %>'
@@ -95,7 +95,7 @@ app.config(['$routeProvider', function ($routeProvider) {
         controller: 'NotificationsController',
         templateUrl: 'views/notifications.html?<%= revision %>'
     }).when('/oidc', {
-        redirectTo: '/usersettings'
+        redirectTo: '/user-directory'
     }).when('/settings', {
         controller: 'SettingsController',
         templateUrl: 'views/settings.html?<%= revision %>'
@@ -120,16 +120,16 @@ app.config(['$routeProvider', function ($routeProvider) {
 app.filter('notificationTypeToColor', function () {
     return function (n) {
         switch (n.type) {
-        case NOTIFICATION_TYPES.ALERT_REBOOT:
-        case NOTIFICATION_TYPES.ALERT_APP_OOM:
-        case NOTIFICATION_TYPES.ALERT_MAIL_STATUS:
-        case NOTIFICATION_TYPES.ALERT_CERTIFICATE_RENEWAL_FAILED:
-        case NOTIFICATION_TYPES.ALERT_DISK_SPACE:
-        case NOTIFICATION_TYPES.ALERT_BACKUP_CONFIG:
-        case NOTIFICATION_TYPES.ALERT_BACKUP_FAILED:
+        case NOTIFICATION_TYPES.REBOOT:
+        case NOTIFICATION_TYPES.APP_OOM:
+        case NOTIFICATION_TYPES.MAIL_STATUS:
+        case NOTIFICATION_TYPES.CERTIFICATE_RENEWAL_FAILED:
+        case NOTIFICATION_TYPES.DISK_SPACE:
+        case NOTIFICATION_TYPES.BACKUP_CONFIG:
+        case NOTIFICATION_TYPES.BACKUP_FAILED:
             return '#ff4c4c';
-        case NOTIFICATION_TYPES.ALERT_BOX_UPDATE:
-        case NOTIFICATION_TYPES.ALERT_MANUAL_APP_UPDATE:
+        case NOTIFICATION_TYPES.BOX_UPDATE:
+        case NOTIFICATION_TYPES.MANUAL_APP_UPDATE:
             return '#f0ad4e';
         default:
             return '#2196f3';

dashboard/public/translation/da.json

@@ -1596,7 +1596,6 @@
             "uninstall": {
                 "title": "Afinstaller",
                 "description": "Dette vil afinstallere appen med det samme og fjerne alle dens data. Der vil ikke være adgang til webstedet.",
-                "backupWarning": "App-backups fjernes ikke, men ryddes op i henhold til backup-politikken. Du kan genoplive denne app fra en eksisterende app-backup ved hjælp af følgende <a target=\"_blank\" href=\"{{ importBackupDocsLink }}}\">instruktioner</a>.",
                 "uninstallAction": "Afinstaller"
             }
         },

dashboard/public/translation/de.json

@@ -1607,7 +1607,6 @@
                 "description": "Anwendungen können angehalten werden, um Server-Ressourcen zu schonen, anstatt sie zu deinstallieren. Zukünftige Anwendungs-Backups werden keine Änderungen von Anwendungen zwischen jetzt und dem letzten Anwendungs-Backup enthalten. Aus diesem Grund wird empfohlen, vor dem Stoppen der Anwendung ein Backup auszulösen."
             },
             "uninstall": {
-                "backupWarning": "Anwendungs-Backups werden nicht entfernt und auf der Grundlage der Backup-Richtlinie bereinigt. Diese Anwendung kann aus einem bestehenden App-Backup mit den folgenden <a target=\"_blank\" href=\"{{ importBackupDocsLink }}\">Schritten</a> wiederhergestellt werden.",
                 "description": "Dies wird die Anwendung sofort deinstallieren und alle zugehörigen Daten löschen. Die Anwendung steht anschließend nicht mehr zur Verfügung.",
                 "title": "Deinstallieren",
                 "uninstallAction": "Deinstallieren"

dashboard/public/translation/en.json

@@ -643,6 +643,26 @@
                 "tooltip": "This will also preserve the mail and {{ appsLength }} app backup(s)."
             },
             "remotePath": "Remote Path"
+        },
+        "archives": {
+            "title": "App Archive",
+            "location": "Location",
+            "info": "Info"
+        },
+        "archive": {
+            "description": "Deleted archives are cleaned up based on the backup policy."
+        },
+        "deleteArchiveDialog": {
+            "title": "Delete Archive of {{appTitle}} ({{fqdn}})",
+            "description": "After deletion, the archive will be cleaned up based on the backup policy."
+        },
+        "deleteArchive": {
+            "deleteAction": "Delete"
+        },
+        "restoreArchiveDialog": {
+            "title": "Restore from Archive",
+            "description": "This will install {{appId}} at the specified location with backup from {{creationTime}}.",
+            "restoreAction": "Restore {{ dnsOverwrite ? 'and overwrite DNS' : '' }}"
         }
     },
     "branding": {
@@ -680,7 +700,7 @@
             "spamFilter": "Spam filtering",
             "spamFilterOverview": "{{ blacklistCount }} address(es) on the blocklist.",
             "changeDomainProgress": "Changing Email domain:",
-            "solrFts": "Full Text Search (Solr)",
+            "solrFts": "Full Text Search",
             "solrEnabled": "Enabled",
             "solrDisabled": "Disabled",
             "solrRunning": "Running",
@@ -747,9 +767,9 @@
             "sendAction": "Send"
         },
         "solrConfig": {
-            "title": "Full Text Search (Solr)",
-            "description": "Solr can be used to provide fast full-text search for emails. Solr is only run if the <a href=\"/#/services\" target=\"_blank\">mail service</a> has been allocated at least 3GB RAM.",
-            "enableSolrCheckbox": "Enable Full Text Search using Solr",
+            "title": "Full Text Search",
+            "description": "Solr &amp; Tika can be used to provide fast full-text search for emails and attachments. Solr is only run if the <a href=\"/#/services\" target=\"_blank\">mail service</a> has been allocated at least 3GB RAM.",
+            "enableSolrCheckbox": "Enable Full Text Search",
             "notEnoughMemory": "Please allocate at least 3GB to the mail service to enable solr."
         },
         "typeFilterHeader": "All Events",
@@ -1089,7 +1109,9 @@
             "deSecToken": "deSEC Token",
             "gandiTokenType": "Token Type",
             "gandiTokenTypeApiKey": "API Key (Deprecated)",
-            "gandiTokenTypePAT": "Personal Access Token (PAT)"
+            "gandiTokenTypePAT": "Personal Access Token (PAT)",
+            "inwxUsername": "Username",
+            "inwxPassword": "Password"
         },
         "removeDialog": {
             "title": "Really remove {{ domain }}?",
@@ -1113,7 +1135,18 @@
         "nonePending": "All Caught Up!",
         "dismissTooltip": "Dismiss",
         "clearAll": "Clear All",
-        "markAllAsRead": "Mark All as Read"
+        "markAllAsRead": "Mark All as Read",
+        "settings": {
+            "title": "Notification Settings",
+            "backupFailed": "Backup failed",
+            "certificateRenewalFailed": "Certificate renewal failed",
+            "appOutOfMemory": "App ran out of memory",
+            "appUp": "App is online",
+            "appDown": "App is down"
+        },
+        "settingsDialog": {
+            "description": "Manage your personal notification preferences here. Cloudron will send an email for the selected events to your primary email address."
+        }
     },
     "logs": {
         "title": "Logs",
@@ -1642,7 +1675,7 @@
             "recovery": {
                 "title": "Crash Recovery",
                 "description": "If the app is not responding, try restarting the app. If the app is constantly restarting because of a broken plugin or misconfiguration, place the app in recovery mode in order to access the console.\nUse the following <a href=\"{{ docsLink }}\" target=\"_blank\">instructions</a> to get the app running again.",
-                "restartAction": "Restart App",
+                "restartAction": "Restart",
                 "enableRecoveryModeAction": "Enable Recovery Mode",
                 "disableRecoveryModeAction": "Disable Recovery Mode"
             },
@@ -1657,13 +1690,12 @@
             "startStop": {
                 "title": "Start / Stop",
                 "description": "Apps can be stopped to conserve server resources instead of uninstalling. Future app backups will not include any app changes between now and the most recent app backup. For this reason, it is recommended to trigger a backup before stopping the app.",
-                "startAction": "Start App",
-                "stopAction": "Stop App"
+                "startAction": "Start",
+                "stopAction": "Stop"
             },
             "uninstall": {
                 "title": "Uninstall",
-                "description": "This will uninstall the app immediately and remove the app's data. The site will be inaccessible.",
-                "backupWarning": "App backups are not removed and will be cleaned up based on the backup policy. You can resurrect this app from an existing app backup using the following <a target=\"_blank\" href=\"{{ importBackupDocsLink }}\">instructions</a>.",
+                "description": "This will uninstall the app and remove the app's data. Backups will be cleaned up based on the backup policy.",
                 "uninstallAction": "Uninstall"
             }
         },
@@ -1681,7 +1713,7 @@
         },
         "uninstallDialog": {
             "title": "Uninstall {{ app }}",
-            "description": "This will immediately uninstall <b>{{ app }}</b> and remove all its data.",
+            "description": "This will uninstall <b>{{ app }}</b> and remove all its data.",
             "uninstallAction": "Uninstall"
         },
         "domainCollisionDialog": {
@@ -1786,6 +1818,17 @@
             "notes": {
                 "title": "Admin Notes"
             }
+        },
+        "archive": {
+            "title": "Archive",
+            "description": "The latest app backup will be added to the <a href=\"/#backups\">App Archive</a>. The app will be uninstalled, but can be restored from the Backups View. Other backups will be cleaned up based on the backup policy.",
+            "action": "Archive",
+            "latestBackupInfo": "The last backup was created at {{date}}.",
+            "noBackup": "This app has no backup. Archiving requires at least one backup."
+        },
+        "archiveDialog": {
+            "title": "Archive {{app}}",
+            "description": "This will uninstall the app and put the app's latest backup created at {{date}} in the App Archive."
         }
     },
     "login": {
@@ -1940,16 +1983,16 @@
     },
     "oidc": {
         "newClientDialog": {
-            "title": "Add Client",
-            "description": "Add new OpenID connect client settings.",
-            "createAction": "Create"
+            "title": "Add OIDC Client",
+            "description": "Enter new OIDC client settings",
+            "createAction": "Add"
         },
         "client": {
             "name": "Name",
             "id": "Client ID",
             "secret": "Client Secret",
             "signingAlgorithm": "Signing Algorithm",
-            "loginRedirectUri": "Login callback Url (comma separated if more than one)",
+            "loginRedirectUri": "Login callback URLs (comma separated)",
             "logoutRedirectUri": "Logout callback Url (optional)"
         },
         "title": "OpenID Connect Provider",
@@ -1959,7 +2002,7 @@
         },
         "deleteClientDialog": {
             "title": "Really delete client {{ client }}?",
-            "description": "This will disconnect all external OpenID apps from this Cloudron using this client ID."
+            "description": "Deleting this OIDC Client will invalidate any access tokens. Apps using this OIDC Client will not be able to authenticate anymore."
         },
         "env": {
             "discoveryUrl": "Discovery URL",
@@ -1971,7 +2014,7 @@
         },
         "clients": {
             "title": "Clients",
-            "newClient": "New client",
+            "newClient": "New Client",
             "empty": "No clients yet"
         }
     },

dashboard/public/translation/es.json

@@ -1263,7 +1263,6 @@
         },
         "uninstall": {
             "uninstall": {
-                "backupWarning": "Las copias de seguridad de las aplicaciones no se eliminan y se borrarán según la política de copias de seguridad. Puede restaurar esta aplicación a partir de una copia de seguridad de la aplicación existente mediante las siguientes <a target=\"_blank\" href=\"{{ importBackupDocsLink }}\"> instrucciones </a>.",
                 "uninstallAction": "Desinstalar",
                 "title": "Desinstalar",
                 "description": "Esto desinstalará la aplicación inmediatamente y eliminará tus datos. El sitio será inaccesible."

dashboard/public/translation/fr.json

@@ -1119,7 +1119,6 @@
         "firstTimeSetupAction": "Initialisation",
         "uninstall": {
             "uninstall": {
-                "backupWarning": "Les sauvegardes de l'application ne sont pas supprimées lors de la désinstallation, elles seront nettoyées plus tard en fonction de la politique de conservation. Vous pouvez réinstaller cette application depuis une ancienne sauvegarde en suivant ces <a target=\"_blank\" href=\"{{ importBackupDocsLink }}\">instructions</a>.",
                 "description": "Cette action entraînera la désinstallation immédiate de l'application et la suppression de l'ensemble de ses données. Le site sera inaccessible.",
                 "uninstallAction": "Désinstaller",
                 "title": "Désinstaller"

dashboard/public/translation/it.json

@@ -258,7 +258,6 @@
         "uninstall": {
             "uninstall": {
                 "uninstallAction": "Disinstalla",
-                "backupWarning": "I backup delle app non vengono rimossi e verranno puliti in base ai criteri di backup. Puoi ripristinare questa app da un backup esistente dell'app utilizzando le seguenti <a target=\"_blank\" href=\"{{ importBackupDocsLink }}\">istruzioni</a>.",
                 "description": "Questo disinstallerà immediatamente l'app e rimuoverà tutti i suoi dati. Il sito sarà inaccessibile.",
                 "title": "Disinstalla"
             },

dashboard/public/translation/nl.json

@@ -643,6 +643,26 @@
             "title": "Bewerk Backup",
             "label": "Label",
             "remotePath": "Extern pad"
+        },
+        "archives": {
+            "title": "Archieven",
+            "location": "Locatie",
+            "info": "Info"
+        },
+        "archive": {
+            "description": "Verwijderde archieven worden opgeschoond op basis van het backup-beleid."
+        },
+        "deleteArchive": {
+            "deleteAction": "Verwijder"
+        },
+        "deleteArchiveDialog": {
+            "description": "Na verwijdering zal het Archief worden opgeschoond op basis van het backup-beleid.",
+            "title": "Verwijder Archief van {{appTitle}} ({{fqdn}})"
+        },
+        "restoreArchiveDialog": {
+            "title": "Herstel vanuit Archief",
+            "description": "Hiermee installeer je {{appId}} op de aangegeven locatie met de backup van {{creationTime}}.",
+            "restoreAction": "Herstel {{ dnsOverwrite ? 'and overwrite DNS' : '' }}"
         }
     },
     "branding": {
@@ -841,7 +861,9 @@
             "deSecToken": "deSEC Token",
             "gandiTokenType": "Token Type",
             "gandiTokenTypeApiKey": "API Sleutel (Uitgefaseerd)",
-            "gandiTokenTypePAT": "Persoonlijke Toegang Token (PAT)"
+            "gandiTokenTypePAT": "Persoonlijke Toegang Token (PAT)",
+            "inwxUsername": "Gebruikersnaam",
+            "inwxPassword": "Wachtwoord"
         },
         "title": "Domeinen & Certificaten",
         "addDomain": "Domein toevoegen",
@@ -1087,7 +1109,7 @@
                 "title": "Crash herstel",
                 "enableRecoveryModeAction": "Herstelmodus inschakelen",
                 "disableRecoveryModeAction": "Herstelmodus uitschakelen",
-                "restartAction": "App herstarten",
+                "restartAction": "Herstarten",
                 "description": "Indien de app niet reageert, probeer dan een herstart van de app. Indien de app continue herstart vanwege een defecte plug-in of verkeerde configuratie, plaats de app dan in herstel-modus voor toegang tot de Terminal.\nVolg deze <a href=\"{{ docsLink }}\" target=\"_blank\">instructies</a> om de app weer werkend te krijgen.."
             },
             "taskError": {
@@ -1100,15 +1122,14 @@
         "uninstall": {
             "startStop": {
                 "title": "Start / Stop",
-                "startAction": "Start App",
-                "stopAction": "Stop App",
+                "startAction": "Start",
+                "stopAction": "Stop",
                 "description": "Apps kunnen ook gestopt worden in plaats van de-installeren om server capaciteit vrij te maken. Toekomstige app backups bevatten geen wijzigingen tussen nu en de laatste app backup. Start daarom handmatig een backup alvorens de app te stoppen."
             },
             "uninstall": {
                 "title": "De-installeer",
                 "uninstallAction": "De-installeer",
-                "backupWarning": "App backups worden niet verwijderd maar opgeschoond volgens de ingestelde bewaartermijn. Je kunt deze app terugzetten met een bestaande backup, volg hiervoor deze <a target=\"_blank\" href=\"{{ importBackupDocsLink }}\">instructies</a>.",
-                "description": "Hierdoor wordt de app direct verwijderd inclusief alle bijbehorende data. De bijbehorende site wordt onbereikbaar."
+                "description": "Hierdoor wordt de app gedeïnstalleerd inclusief alle bijbehorende data. Backups worden opgeschoond op basis van het backup-beleid."
             }
         },
         "appInfo": {
@@ -1126,7 +1147,7 @@
         "uninstallDialog": {
             "uninstallAction": "De-installeer",
             "title": "De-installeer {{ app }}",
-            "description": "Hiermee de-installeer je direct <b>{{ app }}</b> inclusief alle bijbehorende gegevens."
+            "description": "Hiermee deïnstalleer je <b>{{ app }}</b> inclusief alle bijbehorende gegevens."
         },
         "domainCollisionDialog": {
             "title": "Domeinbotsing",
@@ -1232,6 +1253,17 @@
             "notes": {
                 "title": "Admin Notities"
             }
+        },
+        "archive": {
+            "action": "Archiveer",
+            "latestBackupInfo": "De laatste backup werd gemaakt op {{date}}.",
+            "title": "Archief",
+            "description": "De laatste app backup wordt toegevoegd aan het App Archief. De app wordt gedeïnstalleerd maar kan hersteld worden vanuit het Backup Overzicht. Andere backups worden opgeschoond op basis van het backup-beleid.",
+            "noBackup": "Deze app heeft geen backup. Archiveren vereist minstens één backup."
+        },
+        "archiveDialog": {
+            "title": "Archief {{app}}",
+            "description": "Hiermee wordt de app gedeïnstalleerd en wordt de laatste app backup van {{date}} bewaard in het App Archief."
         }
     },
     "network": {
@@ -1463,7 +1495,18 @@
         "dismissTooltip": "Afwijzen",
         "clearAll": "Alles wissen",
         "nonePending": "Alles bijgewerkt!",
-        "markAllAsRead": "Markeer alles als gelezen"
+        "markAllAsRead": "Markeer alles als gelezen",
+        "settings": {
+            "title": "Notificatie instellingen",
+            "backupFailed": "Backup mislukt",
+            "certificateRenewalFailed": "Vernieuwen certificaten mislukt",
+            "appOutOfMemory": "App had te weinig geheugen",
+            "appUp": "App is offline",
+            "appDown": "App werkt niet"
+        },
+        "settingsDialog": {
+            "description": "Beheer hier je persoonlijke notificatie -instellingen. Cloudron zal een e-mail versturen voor de geselecteerde gebeurtenissen naar je primaire e-mailadres."
+        }
     },
     "logs": {
         "title": "Logbestanden",
@@ -1503,7 +1546,7 @@
     "filemanager": {
         "title": "Bestandsbeheer",
         "removeDialog": {
-            "reallyDelete": "Weet je zeker dat je het volgende wilt verwijderen?"
+            "reallyDelete": "Wil je het echt verwijderen?"
         },
         "newDirectoryDialog": {
             "title": "Nieuwe map",
@@ -1940,16 +1983,16 @@
     },
     "oidc": {
         "newClientDialog": {
-            "title": "Client toevoegen",
-            "description": "Nieuwe OpenID Connect client instellingen toevoegen.",
-            "createAction": "Aanmaken"
+            "title": "OIDC Client toevoegen",
+            "description": "Nieuwe OIDC client instellingen invoeren",
+            "createAction": "Toevoegen"
         },
         "client": {
             "name": "Naam",
             "id": "Client ID",
             "secret": "Client geheim",
             "signingAlgorithm": "Ondertekeningsalgoritme",
-            "loginRedirectUri": "Login callback URL (met komma gescheiden indien meer dan één)",
+            "loginRedirectUri": "Login callback URLs (met komma gescheiden)",
             "logoutRedirectUri": "Logout callback URL (optioneel)"
         },
         "title": "OpenID Connect aanbieder",
@@ -1959,7 +2002,7 @@
         },
         "deleteClientDialog": {
             "title": "Weet je zeker dat je Client {{ client }} wilt verwijderen?",
-            "description": "Hiermee worden alle externe OpenID apps met dit Client ID losgekoppeld."
+            "description": "Door het verwijderen van deze OIDC Client worden toegang tokens ongeldig. Apps die deze OIDC Client gebruiken kunnen zich niet meer authenticeren."
         },
         "env": {
             "discoveryUrl": "Discovery URL",

dashboard/public/translation/ru.json

@@ -780,8 +780,7 @@
             "uninstall": {
                 "title": "Удаление",
                 "description": "Данное действие приведёт к полному удалению приложения и его данных. Сайт перестанет быть доступным.",
-                "uninstallAction": "Удалить",
-                "backupWarning": "Резервные копии приложения не удаляются, но очищаются согласно политике хранения. Вы можете восстановить приложение при помощи существующих резервных копий, используя <a target=\"_blank\" href=\"{{ importBackupDocsLink }}\">инструкцию</a>."
+                "uninstallAction": "Удалить"
             }
         },
         "appInfo": {
@@ -889,7 +888,7 @@
             "noApps": "Без приложений",
             "tooltipDownloadBackupConfig": "Скачать конфигурацию резервной копии",
             "cleanupBackups": "Очистить резервные копии",
-            "backupNow": "Создать резервную копию",
+            "backupNow": "Создать копию",
             "tooltipEditBackup": "Редактировать резервную копию",
             "tooltipPreservedBackup": "Резервная копия будет сохранена"
         },
@@ -1438,7 +1437,9 @@
             "deSecToken": "deSEC Токен",
             "gandiTokenType": "Тип токена",
             "gandiTokenTypeApiKey": "API Ключ (Устарело)",
-            "gandiTokenTypePAT": "Персональный токен доступа (PAT)"
+            "gandiTokenTypePAT": "Персональный токен доступа (PAT)",
+            "inwxUsername": "Имя пользователя",
+            "inwxPassword": "Пароль"
         },
         "addDomain": "Добавить домен",
         "removeDialog": {
@@ -1564,7 +1565,7 @@
             "mtime": "Изменён"
         },
         "removeDialog": {
-            "reallyDelete": "Вы действительно хотите удалить выбранные файлы?"
+            "reallyDelete": "Действительно удалить?"
         },
         "extractDialog": {
             "title": "Распаковываем {{ fileName }}",

dashboard/public/translation/vi.json

@@ -1486,7 +1486,6 @@
         "uninstall": {
             "uninstall": {
                 "uninstallAction": "Xoá",
-                "backupWarning": "Các bản sao lưu app sẽ không được xoá ngay mà sẽ dựa vào lịch trình sao lưu được định sẵn. Bạn có thể hồi sinh app từ một bản sao lưu hiện có bằng những <a target=\"_blank\" href=\"{{ importBackupDocsLink }}\">hướng dẫn sau đây</a>.",
                 "description": "Việc này sẽ xóa app ngay lập tức và tất cả dữ liệu. Trang sẽ không còn truy cập được sau khi xóa.",
                 "title": "Xoá"
             },

dashboard/public/translation/zh_Hans.json

@@ -1503,8 +1503,7 @@
             "uninstall": {
                 "description": "将会卸载此应用，并删除所有数据。卸载后该应用将不可用。",
                 "title": "卸载",
-                "uninstallAction": "卸载",
-                "backupWarning": "应用的备份会按照备份政策保留指定的天数，而不会立即被删除。你可以按照 <a target=\"_blank\" href=\"{{ importBackupDocsLink }}\">此步骤</a>从现存的应用备份中恢复该应用。"
+                "uninstallAction": "卸载"
             }
         },
         "importBackupDialog": {

dashboard/public/views/app.html

@@ -13,7 +13,7 @@
       </div>
       <div class="modal-footer">
         <button type="button" class="btn btn-default" data-dismiss="modal">{{ 'main.dialog.close' | tr }}</button>
-        <a class="btn btn-success" ng-href="{{ postInstallMessage.confirmed ? ('https://' + app.fqdn) : '' }}" target="_blank" ng-disabled="!postInstallMessage.confirmed" ng-click="postInstallMessage.submit()" ng-show="postInstallMessage.openApp">{{ 'app.appInfo.openAction' | tr:{ app: app.manifest.title } }}</a>
+        <a class="btn btn-success" ng-href="{{ 'https://' + app.fqdn }}" target="_blank" ng-click="postInstallMessage.submit()" ng-show="postInstallMessage.openApp">{{ 'app.appInfo.openAction' | tr:{ app: app.manifest.title } }}</a>
       </div>
     </div>
   </div>
@@ -47,12 +47,8 @@
         <div ng-show="appPostInstallConfirm.app.manifest.documentationUrl" ng-bind-html="'app.appInfo.appDocsUrl' | tr:{ docsUrl: appPostInstallConfirm.app.manifest.documentationUrl, title: appPostInstallConfirm.app.manifest.title, forumUrl: (appPostInstallConfirm.app.manifest.forumUrl || 'https://forum.cloudron.io') }"></div>
       </div>
       <div class="modal-footer">
-        <div class="form-group pull-left">
-          <input type="checkbox" id="appPostInstallConfirmCheckbox" ng-model="appPostInstallConfirm.confirmed">
-          <label class="control-label" for="appPostInstallConfirmCheckbox">{{ 'app.appInfo.postInstallConfirmCheckbox' | tr }}</label>
-        </div>
         <button type="button" class="btn btn-default" data-dismiss="modal">{{ 'main.dialog.close' | tr }}</button>
-        <a class="btn btn-success" ng-href="{{ appPostInstallConfirm.confirmed ? ('https://' + appPostInstallConfirm.app.fqdn) : '' }}" target="_blank" ng-disabled="!appPostInstallConfirm.confirmed" ng-click="appPostInstallConfirm.submit()">{{ 'app.appInfo.openAction' | tr:{ app: appPostInstallConfirm.app.manifest.title } }}</a>
+        <a class="btn btn-success" ng-href="{{ 'https://' + appPostInstallConfirm.app.fqdn }}" target="_blank" ng-click="appPostInstallConfirm.submit()">{{ 'app.appInfo.openAction' | tr:{ app: appPostInstallConfirm.app.manifest.title } }}</a>
       </div>
     </div>
   </div>
@@ -177,6 +173,24 @@
   </div>
 </div>
 
+<!-- Modal archive app -->
+<div class="modal fade" id="archiveModal" tabindex="-1" role="dialog">
+  <div class="modal-dialog">
+    <div class="modal-content">
+      <div class="modal-header">
+        <h4 class="modal-title">{{ 'app.archiveDialog.title' | tr:{ app: (app.label || app.fqdn) } }}</h4>
+      </div>
+      <div class="modal-body">
+        <p ng-bind-html="'app.archiveDialog.description' | tr:{ date: (uninstall.latestBackup.creationTime | prettyLongDate) }"></p>
+      </div>
+      <div class="modal-footer">
+        <button type="button" class="btn btn-default" data-dismiss="modal">{{ 'main.dialog.cancel' | tr }}</button>
+        <button type="button" class="btn btn-danger" ng-click="uninstall.submit('archive')" ng-disabled="uninstall.busy"><i class="fa fa-circle-notch fa-spin" ng-show="uninstall.busy"></i> {{ 'app.archive.action' | tr }}</button>
+      </div>
+    </div>
+  </div>
+</div>
+
 <!-- Modal uninstall app -->
 <div class="modal fade" id="uninstallModal" tabindex="-1" role="dialog">
   <div class="modal-dialog">
@@ -189,7 +203,7 @@
       </div>
       <div class="modal-footer">
         <button type="button" class="btn btn-default" data-dismiss="modal">{{ 'main.dialog.cancel' | tr }}</button>
-        <button type="button" class="btn btn-danger" ng-click="uninstall.submit()" ng-disabled="uninstall.busy"><i class="fa fa-circle-notch fa-spin" ng-show="uninstall.busy"></i> {{ 'app.uninstallDialog.uninstallAction' | tr }}</button>
+        <button type="button" class="btn btn-danger" ng-click="uninstall.submit('uninstall')" ng-disabled="uninstall.busy"><i class="fa fa-circle-notch fa-spin" ng-show="uninstall.busy"></i> {{ 'app.uninstallDialog.uninstallAction' | tr }}</button>
       </div>
     </div>
   </div>
@@ -773,7 +787,7 @@
           <div class="col-xs-4">
             <span class="text-muted">{{ 'app.updates.info.description' | tr }}</span>
           </div>
-          <div class="col-xs-8 text-right">
+          <div class="col-xs-8 text-right no-wrap-scroll">
             <span ng-show="app.appStoreId">{{ app.manifest.title }} {{ app.upstreamVersion }}</span>
             <span ng-show="!app.appStoreId">{{ app.manifest.dockerImage }}</span>
           </div>
@@ -1136,7 +1150,7 @@
             <form role="form" name="resourcesForm" ng-submit="resources.submitCpuQuota()" autocomplete="off">
               <fieldset>
                 <div class="form-group">
-                  <label class="control-label" for="cpuQuota">{{ 'app.resources.cpu.title' | tr }} <sup><a ng-href="https://docs.cloudron.io/apps/#cpu-quota" class="help" target="_blank"><i class="fa fa-question-circle"></i></a></sup> : <b>{{ resources.cpuQuota + ' %' }}</b></label>
+                  <label class="control-label" for="cpuQuota">{{ 'app.resources.cpu.title' | tr }} <sup><a ng-href="https://docs.cloudron.io/apps/#cpu-limit" class="help" target="_blank"><i class="fa fa-question-circle"></i></a></sup> : <b>{{ resources.cpuQuota + ' %' }}</b></label>
                   <p>{{ 'app.resources.cpu.description' | tr }}</p>
                   <input type="range" id="cpuQuota" ng-model="resources.cpuQuota" step="1" min="1" max="100"/>
                   <datalist id="cpuQuotaTicks">
@@ -1154,6 +1168,26 @@
             <button class="btn btn-outline btn-primary pull-right" ng-click="resources.submitCpuQuota()" ng-disabled="resources.cpuQuota === resources.currentCpuQuota || resourcesForm.$invalid || resources.busy || app.error || app.taskId" tooltip-enable="app.error || app.taskId" uib-tooltip="{{ app.error ? 'App is in error state' : 'App is busy' }}">{{ 'app.resources.cpu.setAction' | tr }}</button>
           </div>
         </div>
+        <hr/>
+        <div class="row">
+          <div class="col-md-12">
+            <form role="form" name="devicesForm" ng-submit="resources.submitDevices()" autocomplete="off">
+              <fieldset>
+                <div class="form-group">
+                  <label class="control-label" for="devicesInput">Devices <sup><a ng-href="https://docs.cloudron.io/apps/#devices" class="help" target="_blank"><i class="fa fa-question-circle"></i></a></sup></label>
+                  <p>Comma serparated list of devices mounted into the app</p>
+                  <input type="text" class="form-control" ng-class="{ 'has-error': resources.error.devices }" id="devicesInput" ng-model="resources.devices" placeholder="/dev/ttyUSB0, /dev/hidraw0, ..." ng-disabled="resources.busy"/>
+                  <span class="text-danger" ng-show="resources.error.devices">{{ resources.error.devices }}</span>
+                </div>
+              </fieldset>
+            </form>
+          </div>
+        </div>
+        <div class="row">
+          <div class="col-md-12 text-right">
+            <button class="btn btn-outline btn-primary pull-right" ng-click="resources.submitDevices()" ng-disabled="devicesForm.$invalid || resources.busy || app.error || app.taskId" tooltip-enable="app.error || app.taskId" uib-tooltip="{{ app.error ? 'App is in error state' : 'App is busy' }}">Set Devices</button>
+          </div>
+        </div>
       </div>
 
       <div class="card" ng-show="view === 'services'">
@@ -1701,13 +1735,22 @@
             </button>
           </div>
         </div>
+        <hr ng-if="app.type !== APP_TYPES.PROXIED"/>
+        <div class="row" ng-if="app.type !== APP_TYPES.PROXIED">
+          <div class="col-md-12">
+            <label class="control-label">{{ 'app.archive.title' | tr }}</label>
+            <p ng-bind-html="'app.archive.description' | tr"></p>
+            <p class="text-bold text-success" ng-show="uninstall.latestBackup" ng-bind-html="'app.archive.latestBackupInfo' | tr:{ date: (uninstall.latestBackup.creationTime | prettyLongDate) }"></p>
+            <p class="text-bold text-warning" ng-show="!uninstall.latestBackup" ng-bind-html="'app.archive.noBackup' | tr"></p>
+            <button ng-disabled="!uninstall.latestBackup" class="btn btn-default pull-right" ng-click="uninstall.ask('archive')">{{ 'app.archive.action' | tr }}</button>
+          </div>
+        </div>
         <hr/>
         <div class="row">
           <div class="col-md-12">
             <label class="control-label">{{ 'app.uninstall.uninstall.title' | tr }}</label>
             <p>{{ 'app.uninstall.uninstall.description' | tr }}</p>
-            <p ng-bind-html="'app.uninstall.uninstall.backupWarning' | tr:{ importBackupDocsLink: 'https://docs.cloudron.io/backups/#import-app-backup' }"></p>
-            <button class="btn btn-danger pull-right" ng-click="uninstall.ask()">{{ 'app.uninstall.uninstall.uninstallAction' | tr }}</button>
+            <button class="btn btn-danger pull-right" ng-click="uninstall.ask('uninstall')">{{ 'app.uninstall.uninstall.uninstallAction' | tr }}</button>
           </div>
         </div>
       </div>

dashboard/public/views/app.js

@@ -1,6 +1,6 @@
 'use strict';
 
-/* global angular, localStorage, document, FileReader */
+/* global angular */
 /* global $ */
 /* global async */
 /* global RSTATES */
@@ -81,12 +81,10 @@ angular.module('Application').controller('AppController', ['$scope', '$location'
     $scope.appPostInstallConfirm = {
         app: {},
         message: '',
-        confirmed: false,
 
         show: function (app) {
             $scope.appPostInstallConfirm.app = app;
             $scope.appPostInstallConfirm.message = app.manifest.postInstallMessage;
-            $scope.appPostInstallConfirm.confirmed = false;
 
             $('#appPostInstallConfirmModal').modal('show');
 
@@ -94,8 +92,6 @@ angular.module('Application').controller('AppController', ['$scope', '$location'
         },
 
         submit: function () {
-            if (!$scope.appPostInstallConfirm.confirmed) return;
-
             $scope.appPostInstallConfirm.app.pendingPostInstallConfirmation = false;
             delete localStorage['confirmPostInstall_' + $scope.appPostInstallConfirm.app.id];
 
@@ -104,11 +100,9 @@ angular.module('Application').controller('AppController', ['$scope', '$location'
     };
 
     $scope.postInstallMessage = {
-        confirmed: false,
         openApp: false,
 
         show: function (openApp) {
-            $scope.postInstallMessage.confirmed = false;
             $scope.postInstallMessage.openApp = !!openApp;
 
             if (!$scope.app.manifest.postInstallMessage) return;
@@ -116,8 +110,6 @@ angular.module('Application').controller('AppController', ['$scope', '$location'
         },
 
         submit: function () {
-            if (!$scope.postInstallMessage.confirmed) return;
-
             $scope.app.pendingPostInstallConfirmation = false;
             delete localStorage['confirmPostInstall_' + $scope.app.id];
 
@@ -613,6 +605,7 @@ angular.module('Application').controller('AppController', ['$scope', '$location'
 
         currentCpuQuota: 0,
         cpuQuota: 0,
+        devices: '',
 
         show: function () {
             var app = $scope.app;
@@ -644,6 +637,8 @@ angular.module('Application').controller('AppController', ['$scope', '$location'
                 $scope.resources.memoryLimit = $scope.resources.currentMemoryLimit;
                 $scope.resources.busy = false;
             }, 500);
+
+            $scope.resources.devices = Object.keys(app.devices).join(', ');
         },
 
         submitMemoryLimit: function () {
@@ -687,6 +682,32 @@ angular.module('Application').controller('AppController', ['$scope', '$location'
                 });
             });
         },
+
+        submitDevices: function () {
+            $scope.resources.busy = true;
+            $scope.resources.error = {};
+
+            const devices = {};
+            $scope.resources.devices.split(',').forEach(d => {
+                if (!d.trim()) return;
+                devices[d.trim()] = {};
+            });
+
+            Client.configureApp($scope.app.id, 'devices', { devices }, function (error) {
+                if (error && error.statusCode === 400) {
+                    $scope.resources.error.devices = error.message;
+                    return $scope.resources.busy = false;
+                } else if (error) {
+                    return Client.error(error);
+                }
+
+                refreshApp($scope.app.id, function (error) {
+                    if (error) return Client.error(error);
+
+                    $timeout(function () { $scope.resources.busy = false; }, 1000);
+                });
+            });
+        },
     };
 
     $scope.services = {
@@ -1706,6 +1727,7 @@ angular.module('Application').controller('AppController', ['$scope', '$location'
         error: {},
         busyRunState: false,
         startButton: false,
+        latestBackup: null,
 
         toggleRunState: function (confirmStop) {
             if (confirmStop && $scope.app.runState !== RSTATES.STOPPED) {
@@ -1731,26 +1753,44 @@ angular.module('Application').controller('AppController', ['$scope', '$location'
 
         show: function () {
             $scope.uninstall.error = {};
+
+            $scope.uninstall.latestBackup = null;
+
+            Client.getAppBackups($scope.app.id, function (error, backups) {
+                if (!error && backups.length) $scope.uninstall.latestBackup = backups[0];
+            });
         },
 
-        ask: function () {
-            $('#uninstallModal').modal('show');
+        ask: function (what) {
+            if (what === 'uninstall') {
+                $('#uninstallModal').modal('show');
+            } else {
+                $('#archiveModal').modal('show');
+            }
         },
 
-        submit: function () {
+        submit: function (what) {
             $scope.uninstall.busy = true;
 
             var NOOP = function (next) { return next(); };
             var stopAppTask = $scope.app.taskId ? Client.stopTask.bind(null, $scope.app.taskId) : NOOP;
 
             stopAppTask(function () { // ignore error
-                Client.uninstallApp($scope.app.id, function (error) {
+                const func = what === 'uninstall' ?
+                    Client.uninstallApp.bind(null, $scope.app.id) :
+                    Client.archiveApp.bind(Client, $scope.app.id, $scope.uninstall.latestBackup.id);
+
+                func(function (error) {
                     if (error && error.statusCode === 402) { // unpurchase failed
                         Client.error('Relogin to Cloudron App Store');
                     } else if (error) {
                         Client.error(error);
                     } else {
-                        $('#uninstallModal').modal('hide');
+                        if (what === 'uninstall') {
+                            $('#uninstallModal').modal('hide');
+                        } else {
+                            $('#archiveModal').modal('hide');
+                        }
                         $location.path('/apps');
                     }
 
@@ -2243,7 +2283,7 @@ angular.module('Application').controller('AppController', ['$scope', '$location'
             }
         });
 
-        var filename = 'app-backup-config-' + (new Date()).toISOString().replace(/:|T/g,'-').replace(/\..*/,'') + ' (' + $scope.app.fqdn + ')' + '.json';
+        const filename = `${$scope.app.fqdn}-backup-config-${(new Date(backup.creationTime)).toISOString().split('T')[0]}.json`;
         download(filename, JSON.stringify(tmp, null, 4));
     };
 

dashboard/public/views/apps.html

@@ -252,7 +252,7 @@
                   <i class="fas fa-envelope" ng-show="app.manifest.addons.email" uib-tooltip="{{ 'apps.auth.email' | tr }}"></i>
                 </div>
               </td>
-              <td class="elide-table-cell text-right">
+              <td class="text-right" style="vertical-align: middle; white-space: nowrap;">
                 <span ng-show="isOperator(app)">
                   <a class="btn btn-xs btn-success" style="padding: 1px 7px;" ng-show="config.update[app.id].manifest.version && config.update[app.id].manifest.version !== app.manifest.version && (app | installSuccess) && !(app.error || app.runState === 'stopped')" ng-href="#/app/{{ app.id}}/updates" uib-tooltip="Update Available"><i class="fa fa-arrow-up"></i></a>
 

dashboard/public/views/apps.js

@@ -4,7 +4,6 @@
 /* global $:false */
 /* global APP_TYPES */
 /* global onAppClick */
-/* global localStorage, document, FileReader */
 
 angular.module('Application').controller('AppsController', ['$scope', '$translate', '$interval', '$location', 'Client', function ($scope, $translate, $interval, $location, Client) {
     var ALL_DOMAINS_DOMAIN = { _alldomains: true, domain: 'All Domains' }; // dummy record for the single select filter

dashboard/public/views/appstore.html

@@ -140,29 +140,6 @@
               <br/>
             </div>
 
-            <div class="hide">
-              <label class="control-label" for="appInstallCertificateInput">Certificate (optional)</label>
-              <div class="has-error text-center" ng-show="appInstall.error.cert">{{ appInstall.error.cert }}</div>
-              <div class="form-group" ng-class="{ 'has-error': !appInstallForm.certificate.$dirty && appInstall.error.cert }">
-                <div class="input-group">
-                  <input type="file" id="appInstallCertificateFileInput" style="display:none"/>
-                  <input type="text" class="form-control" placeholder="Certificate" ng-model="appInstall.certificateFileName" id="appInstallCertificateInput" name="certificate" onclick="getElementById('appInstallCertificateFileInput').click();" style="cursor: pointer;" ng-required="appInstall.keyFileName">
-                  <span class="input-group-addon">
-                    <i class="fa fa-upload" onclick="getElementById('appInstallCertificateFileInput').click();"></i>
-                  </span>
-                </div>
-              </div>
-              <div class="form-group" ng-class="{ 'has-error': !appInstallForm.key.$dirty && appInstall.error.cert }">
-                <div class="input-group">
-                  <input type="file" id="appInstallKeyFileInput" style="display:none"/>
-                  <input type="text" class="form-control" placeholder="Key" ng-model="appInstall.keyFileName" id="appInstallKeyInput" name="key" onclick="getElementById('appInstallKeyFileInput').click();" style="cursor: pointer;" ng-required="appInstall.certificateFileName">
-                  <span class="input-group-addon">
-                    <i class="fa fa-upload" onclick="getElementById('appInstallKeyFileInput').click();"></i>
-                  </span>
-                </div>
-              </div>
-            </div>
-
             <input class="ng-hide" type="submit" ng-disabled="(appInstall.accessRestrictionOption === 'groups' && !appInstall.isAccessRestrictionValid()) || !appInstall.accessRestrictionOption || appInstallForm.$invalid || busy"/>
           </form>
         </div>

dashboard/public/views/appstore.js

@@ -125,8 +125,6 @@ angular.module('Application').controller('AppStoreController', ['$scope', '$tran
         ports: {},
         portsEnabled: {},
         mediaLinks: [],
-        certificateFile: null,
-        certificateFileName: '',
         keyFile: null,
         keyFileName: '',
         accessRestrictionOption: '',
@@ -151,8 +149,6 @@ angular.module('Application').controller('AppStoreController', ['$scope', '$tran
             $scope.appInstall.ports = {};
             $scope.appInstall.state = 'appInfo';
             $scope.appInstall.mediaLinks = [];
-            $scope.appInstall.certificateFile = null;
-            $scope.appInstall.certificateFileName = '';
             $scope.appInstall.keyFile = null;
             $scope.appInstall.keyFileName = '';
             $scope.appInstall.accessRestrictionOption = '';
@@ -276,8 +272,6 @@ angular.module('Application').controller('AppStoreController', ['$scope', '$tran
                 secondaryDomains: secondaryDomains,
                 ports: finalPorts,
                 accessRestriction: finalAccessRestriction,
-                cert: $scope.appInstall.certificateFile,
-                key: $scope.appInstall.keyFile,
                 sso: !$scope.appInstall.optionalSso ? undefined : ($scope.appInstall.accessRestrictionOption !== 'nosso'),
             };
 
@@ -339,7 +333,7 @@ angular.module('Application').controller('AppStoreController', ['$scope', '$tran
                     return;
                 }
 
-                Client.installApp($scope.appInstall.app.id, $scope.appInstall.app.manifest, $scope.appInstall.app.title, data, function (error, newAppId) {
+                Client.installApp($scope.appInstall.app.id, $scope.appInstall.app.manifest, data, function (error, newAppId) {
                     if (error) {
                         var errorMessage = error.message.toLowerCase();
 
@@ -365,15 +359,7 @@ angular.module('Application').controller('AppStoreController', ['$scope', '$tran
                                 $scope.appInstall.error.other = error.message;
                             }
                         } else if (error.statusCode === 400) {
-                            if (errorMessage.indexOf('cert') !== -1) {
-                                $scope.appInstall.error.cert = error.message;
-                                $scope.appInstall.certificateFileName = '';
-                                $scope.appInstall.certificateFile = null;
-                                $scope.appInstall.keyFileName = '';
-                                $scope.appInstall.keyFile = null;
-                            } else {
-                                $scope.appInstall.error.other = error.message;
-                            }
+                            $scope.appInstall.error.other = error.message;
                         } else {
                             $scope.appInstall.error.other = error.message;
                         }
@@ -568,34 +554,6 @@ angular.module('Application').controller('AppStoreController', ['$scope', '$tran
         });
     };
 
-    document.getElementById('appInstallCertificateFileInput').onchange = function (event) {
-        $scope.$apply(function () {
-            $scope.appInstall.certificateFile = null;
-            $scope.appInstall.certificateFileName = event.target.files[0].name;
-
-            var reader = new FileReader();
-            reader.onload = function (result) {
-                if (!result.target || !result.target.result) return console.error('Unable to read local file');
-                $scope.appInstall.certificateFile = result.target.result;
-            };
-            reader.readAsText(event.target.files[0]);
-        });
-    };
-
-    document.getElementById('appInstallKeyFileInput').onchange = function (event) {
-        $scope.$apply(function () {
-            $scope.appInstall.keyFile = null;
-            $scope.appInstall.keyFileName = event.target.files[0].name;
-
-            var reader = new FileReader();
-            reader.onload = function (result) {
-                if (!result.target || !result.target.result) return console.error('Unable to read local file');
-                $scope.appInstall.keyFile = result.target.result;
-            };
-            reader.readAsText(event.target.files[0]);
-        });
-    };
-
     $scope.showAppNotFound = function (appId, version) {
         $scope.appNotFound.appId = appId;
         $scope.appNotFound.version = version || 'latest';

dashboard/public/views/backups.html

@@ -450,6 +450,115 @@
   </div>
 </div>
 
+<!-- Modal archive restore -->
+<div class="modal fade" id="restoreArchiveModal" tabindex="-1" role="dialog">
+  <div class="modal-dialog">
+    <div class="modal-content">
+      <div class="modal-header">
+        <h4 class="modal-title">{{ 'backups.restoreArchiveDialog.title' | tr }}</h4>
+      </div>
+      <div class="modal-body" style="padding: 0 15px">
+        <p ng-bind-html="'backups.restoreArchiveDialog.description' | tr:{ appId: archiveRestore.app.manifest.id, fqdn: archiveRestore.app.fqdn, creationTime: (archiveRestore.archive.creationTime | prettyLongDate) }"></p>
+        <form role="form" ng-submit="archiveRestore.submit()" autocomplete="off">
+          <fieldset>
+            <div class="form-group" ng-class="{ 'has-error': archiveRestore.error.location.fqdn === archiveRestore.subdomain + '.' + archiveRestore.domain.domain }">
+              <label class="control-label" for="cloneLocationInput">{{ 'app.cloneDialog.location' | tr }}</label>
+              <div ng-show="archiveRestore.error.location.fqdn === archiveRestore.subdomain + '.' + archiveRestore.domain.domain"><small>{{ archiveRestore.error.location.message }}</small></div>
+              <div class="input-group form-inline">
+                <input type="text" class="form-control" ng-model="archiveRestore.subdomain" id="cloneLocationInput" name="location" placeholder="{{ 'appstore.installDialog.locationPlaceholder' | tr }}" autofocus>
+                <div class="input-group-btn">
+                  <button type="button" class="btn btn-default dropdown-toggle" data-toggle="dropdown">
+                    <span>{{ '.' + archiveRestore.domain.domain }}</span>
+                    <span class="caret"></span>
+                  </button>
+                  <ul class="dropdown-menu dropdown-menu-right" role="menu">
+                    <li ng-repeat="domain in domains">
+                      <a href="" ng-click="archiveRestore.domain = domain">{{ domain.domain }}</a>
+                    </li>
+                  </ul>
+                </div>
+              </div>
+            </div>
+
+            <div class="has-error text-center" ng-show="archiveRestore.error.secondaryDomain">{{ archiveRestore.error.secondaryDomain }}</div>
+            <div ng-repeat="(env, info) in archiveRestore.app.manifest.httpPorts">
+              <ng-form name="secondaryDomainInfo_form">
+                <div class="form-group" ng-class="{ 'has-error': (!secondaryDomainInfo_form.itemName{{$index}}.$dirty && archiveRestore.error.secondaryDomain) || (secondaryDomainInfo_form.itemName{{$index}}.$dirty && secondaryDomainInfo_form.itemName{{$index}}.$invalid) || (archiveRestore.error.location.fqdn === archiveRestore.secondaryDomains[env].subdomain + '.' + archiveRestore.secondaryDomains[env].domain.domain) }">
+                  <label class="control-label" for="secondaryDomainInput{{env}}">
+                    {{ info.title }}
+                    <sup>
+                      <a popover-placement="top-right" popover-trigger="outsideClick" uib-popover="{{info.description}}"><i class="fa fa-question-circle"></i></a>
+                    </sup>
+                  </label>
+
+                  <div ng-show="archiveRestore.error.location.fqdn === archiveRestore.secondaryDomains[env].subdomain + '.' + archiveRestore.secondaryDomains[env].domain.domain"><small>{{ archiveRestore.error.location.message }}</small></div>
+                  <div class="input-group form-inline">
+                    <input type="text" class="form-control" ng-model="archiveRestore.secondaryDomains[env].subdomain" name="location{{$index}}" placeholder="{{ 'app.location.locationPlaceholder' | tr }}" autofocus>
+
+                    <div class="input-group-btn">
+                      <button type="button" class="btn btn-default dropdown-toggle" data-toggle="dropdown">
+                        <span>.{{ archiveRestore.secondaryDomains[env].domain.domain }}</span>
+                        <span class="caret"></span>
+                      </button>
+                      <ul class="dropdown-menu dropdown-menu-right" role="menu">
+                        <li ng-repeat="domain in domains">
+                          <a href="" ng-click="archiveRestore.secondaryDomains[env].domain = domain">{{ domain.domain }}</a>
+                        </li>
+                      </ul>
+                    </div>
+                  </div>
+                </div>
+              </ng-form>
+            </div>
+
+            <p class="text-small text-warning" ng-show="archiveRestore.domain.provider === 'noop' || archiveRestore.domain.provider === 'manual'" ng-bind-html="'appstore.installDialog.manualWarning' | tr:{ location: ((archiveRestore.subdomain ? archiveRestore.subdomain + '.' : '') + archiveRestore.domain.domain) }"></p>
+
+            <div class="has-error text-center" ng-show="archiveRestore.error.port">{{ archiveRestore.error.port }}</div>
+            <div ng-repeat="(env, info) in archiveRestore.portInfo">
+              <ng-form name="portInfo_form">
+                <div class="form-group" ng-class="{ 'has-error': (!archiveRestore.itemName{{$index}}.$dirty && archiveRestore.error.port) || (portInfo_form.itemName{{$index}}.$dirty && portInfo_form.itemName{{$index}}.$invalid) }">
+                  <label class="control-label" for="inputPortInfo{{env}}"><input type="checkbox" ng-model="archiveRestore.portsEnabled[env]">
+                    {{ info.title }}
+                    <sup>
+                      <a popover-placement="top-right" popover-trigger="outsideClick" uib-popover="{{info.description}}"><i class="fa fa-question-circle"></i></a>
+                    </sup>
+                    <small style="padding-left: 5px;" ng-show="info.readOnly">{{ 'appstore.installDialog.portReadOnly' | tr }}</small>
+                  </label>
+                  <input type="number" class="form-control" ng-model="archiveRestore.ports[env]" ng-disabled="!archiveRestore.portsEnabled[env]" ng-readonly="info.readOnly" id="inputPortInfo{{env}}" later-name="itemName{{$index}}" min="{{hostPortMin}}" max="{{hostPortMax}}" required>
+                  <p class="text-small text-warning text-bold" ng-show="archiveRestore.domain.provider === 'cloudflare'">{{ 'appstore.installDialog.cloudflarePortWarning' | tr }} </p>
+                </div>
+              </ng-form>
+            </div>
+          </fieldset>
+        </form>
+      </div>
+
+      <div class="modal-footer">
+        <button type="button" class="btn btn-default" data-dismiss="modal">{{ 'main.dialog.cancel' | tr }}</button>
+        <button type="button" class="btn btn-success" ng-click="archiveRestore.submit()"><i class="fas fa-history" ng-hide="archiveRestore.busy"></i><i class="fa fa-circle-notch fa-spin" ng-show="archiveRestore.busy"></i> {{ 'backups.restoreArchiveDialog.restoreAction' | tr:{ dnsOverwrite: archiveRestore.needsOverwrite } }}</button>
+      </div>
+    </div>
+  </div>
+</div>
+
+<!-- Modal delete archive -->
+<div class="modal fade" id="archiveDeleteModal" tabindex="-1" role="dialog">
+  <div class="modal-dialog">
+    <div class="modal-content">
+      <div class="modal-header">
+        <h4 class="modal-title">{{ 'backups.deleteArchiveDialog.title' | tr:{ appTitle: archiveDelete.app.manifest.title, fqdn: archiveDelete.app.fqdn } }}</h4>
+      </div>
+      <div class="modal-body">
+        <p>{{ 'backups.deleteArchiveDialog.description' | tr }}</p>
+      </div>
+      <div class="modal-footer">
+        <button type="button" class="btn btn-default" data-dismiss="modal">{{ 'main.dialog.cancel' | tr }}</button>
+        <button type="button" class="btn btn-danger" ng-click="archiveDelete.submit()" ng-disabled="archiveDelete.busy"><i class="fa fa-circle-notch fa-spin" ng-show="archiveDelete.busy"></i> {{ 'backups.deleteArchive.deleteAction' | tr }}</button>
+      </div>
+    </div>
+  </div>
+</div>
+
 <div class="content">
 
   <h1 class="section-header">{{ 'backups.title' | tr }}</h1>
@@ -643,4 +752,57 @@
       </div>
     </div>
   </div>
+
+  <h3 class="section-header">
+    {{ 'backups.archives.title' | tr }}
+  </h3>
+
+  <div class="card card-large">
+    <p ng-bind-html=" 'backups.archive.description' | tr "></p>
+
+    <div class="grid-item-top">
+      <div class="row ng-hide" ng-show="!archiveList.ready">
+        <div class="col-lg-12 text-center">
+          <h2><i class="fa fa-circle-notch fa-spin"></i></h2>
+        </div>
+      </div>
+      <div class="row animateMeOpacity ng-hide" ng-show="archiveList.ready">
+        <div class="col-lg-12">
+          <table class="table table-hover" style="margin: 0;">
+            <thead>
+              <tr>
+                <th style="width: 5%"></th> <!-- icon -->
+                <th style="width: 35%">{{ 'backups.archives.location' | tr }}</th>
+                <th style="width: 35%" class="hide-mobile">{{ 'backups.archives.info' | tr }}</th>
+                <th style="width: 20%">{{ 'main.table.date' | tr }}</th>
+                <th style="width: 5%" class="text-right hide-mobile">{{ 'main.actions' | tr }}</th>
+              </tr>
+            </thead>
+            <tbody>
+              <tr ng-repeat="archive in archiveList.archives">
+                <td>
+                  <img ng-src="{{ archive.iconUrl || 'img/appicon_fallback.png' }}" fallback-icon="img/appicon_fallback.png" onerror="imageErrorHandler(this)" height="48" width="48"/>
+                </td>
+                <td class="hand elide-table-cell" style="text-overflow: ellipsis; white-space: nowrap;" ng-click="archiveRestore.show(archive)">
+                  {{ archive.appConfig.fqdn }}
+                </td>
+                <td class="hand elide-table-cell hide-mobile" style="text-overflow: ellipsis; white-space: nowrap;" ng-click="archiveRestore.show(archive)">
+                  <span uib-tooltip="{{ archive.appConfig.manifest.id }}@{{ archive.appConfig.manifest.version }}">{{ archive.appConfig.manifest.title }}</span>
+                </td>
+                <td class="hand elide-table-cell" style="text-overflow: ellipsis; white-space: nowrap;" ng-click="archiveRestore.show(archive)">
+                  {{ archive.creationTime | prettyDate }}
+                </td>
+                <td class="text-right no-wrap hide-mobile" style="vertical-align: middle;">
+                  <button class="btn btn-xs btn-default" ng-click="archiveRestore.show(archive)" uib-tooltip="Restore from Archive"><i class="fas fa-history"></i></button>
+                  <button class="btn btn-xs btn-default" ng-click="downloadConfig(archive, true)" uib-tooltip="{{ 'backups.listing.tooltipDownloadBackupConfig' | tr }}"><i class="fas fa-file-alt"></i></button>
+                  <button class="btn btn-xs btn-danger" ng-click="archiveDelete.ask(archive)" uib-tooltip="Delete Archive"><i class="fa fa-trash-alt"></i></button>
+                </td>
+              </tr>
+            </tbody>
+          </table>
+        </div>
+      </div>
+    </div>
+  </div>
+
 </div>

dashboard/public/views/backups.js

@@ -2,7 +2,7 @@
 
 /* global $, angular, TASK_TYPES, SECRET_PLACEHOLDER, STORAGE_PROVIDERS, BACKUP_FORMATS, APP_TYPES */
 /* global REGIONS_S3, REGIONS_WASABI, REGIONS_DIGITALOCEAN, REGIONS_EXOSCALE, REGIONS_SCALEWAY, REGIONS_LINODE, REGIONS_OVH, REGIONS_IONOS, REGIONS_UPCLOUD, REGIONS_VULTR , REGIONS_CONTABO, REGIONS_HETZNER */
-/* global document, window, FileReader */
+/* global async, ERROR */
 
 angular.module('Application').controller('BackupsController', ['$scope', '$location', '$rootScope', '$timeout', 'Client', function ($scope, $location, $rootScope, $timeout, Client) {
     Client.onReady(function () { if (!Client.getUserInfo().isAtLeastAdmin) $location.path('/'); });
@@ -23,6 +23,8 @@ angular.module('Application').controller('BackupsController', ['$scope', '$locat
     $scope.backupTasks = [];
     $scope.cleanupTasks = [];
 
+    $scope.domains = [];
+
     $scope.s3Regions = REGIONS_S3;
     $scope.wasabiRegions = REGIONS_WASABI;
     $scope.doSpacesRegions = REGIONS_DIGITALOCEAN;
@@ -266,7 +268,210 @@ angular.module('Application').controller('BackupsController', ['$scope', '$locat
         }
     };
 
-    $scope.listBackups = {
+    $scope.archiveList = {
+        ready: false,
+        archives: [],
+
+        fetch: function () {
+            Client.listArchives(function (error, archives) {
+                if (error) Client.error(error);
+                $scope.archiveList.archives = archives;
+                $scope.archiveList.ready = true;
+
+                // ensure we use the full api oprigin
+                $scope.archiveList.archives.forEach(a => {
+                    a.iconUrl = window.cloudronApiOrigin + a.iconUrl;
+                });
+            });
+        },
+    };
+
+    $scope.archiveDelete = {
+        busy: false,
+        error: {},
+        archive: null,
+        app: null, // just for simpler access . it's a fake app object!
+
+        ask: function (archive) {
+            $scope.archiveDelete.busy = false;
+            $scope.archiveDelete.error = {};
+            $scope.archiveDelete.archive = archive;
+            $scope.archiveDelete.app = archive.appConfig;
+            $('#archiveDeleteModal').modal('show');
+        },
+
+        submit: function () {
+            $scope.archiveDelete.busy = true;
+            $scope.archiveDelete.error = {};
+
+            Client.deleteArchive($scope.archiveDelete.archive.id, function (error) {
+                $scope.archiveDelete.busy = false;
+                if (error) return console.error('Unable to delete archive.', error.statusCode, error.message);
+                $scope.archiveList.fetch();
+                $('#archiveDeleteModal').modal('hide');
+            });
+        }
+    };
+
+    // keep in sync with app.js
+    $scope.archiveRestore = {
+        busy: false,
+        error: {},
+
+        archive: null,
+        app: null, // just for simpler access . it's a fake app object!
+
+        subdomain: '',
+        domain: null,
+        secondaryDomains: {},
+        needsOverwrite: false,
+        overwriteDns: false,
+        ports: {},
+        portsEnabled: {},
+        portInfo: {},
+        accessRestriction: { users: [], groups: [] },
+
+        init: function () {
+            Client.getDomains(function (error, domains) {
+                if (error) return console.error('Unable to get domain listing.', error);
+                $scope.domains = domains;
+            });
+        },
+
+        show: function (archive) {
+            $scope.archiveRestore.error = {};
+            $scope.archiveRestore.archive = archive;
+            const manifest = archive.appConfig.manifest;
+
+            $scope.archiveRestore.app = archive.appConfig;
+            $scope.archiveRestore.subdomain = $scope.archiveRestore.app.subdomain;
+            $scope.archiveRestore.domain = $scope.domains.find(function (d) { return $scope.archiveRestore.app.domain === d.domain; }); // try to pre-select the app's domain
+
+            $scope.archiveRestore.needsOverwrite = false;
+            $scope.archiveRestore.overwriteDns = false;
+
+            $scope.archiveRestore.secondaryDomains = {};
+
+            var httpPorts = manifest.httpPorts || {};
+            for (var env2 in httpPorts) {
+                $scope.archiveRestore.secondaryDomains[env2] = {
+                    subdomain: httpPorts[env2].defaultValue || '',
+                    domain: $scope.archiveRestore.domain
+                };
+            }
+            // now fill secondaryDomains with real values, if it exists
+            $scope.archiveRestore.app.secondaryDomains.forEach(function (sd) {
+                $scope.archiveRestore.secondaryDomains[sd.environmentVariable] = {
+                    subdomain: sd.subdomain,
+                    domain: $scope.domains.find(function (d) { return sd.domain === d.domain; })
+                };
+            });
+
+            $scope.archiveRestore.portInfo = angular.extend({}, manifest.tcpPorts, manifest.udpPorts);   // Portbinding map only for information
+            // set default ports
+            for (var env in $scope.archiveRestore.portInfo) {
+                if ($scope.archiveRestore.app.portBindings[env]) { // was enabled in the app
+                    $scope.archiveRestore.ports[env] = $scope.archiveRestore.app.portBindings[env].hostPort;
+                    $scope.archiveRestore.portsEnabled[env] = true;
+                } else {
+                    $scope.archiveRestore.ports[env] = $scope.archiveRestore.portInfo[env].defaultValue || 0;
+                    $scope.archiveRestore.portsEnabled[env] = false;
+                }
+            }
+
+            $('#restoreArchiveModal').modal('show');
+        },
+
+        submit: function () {
+            $scope.archiveRestore.busy = true;
+
+            var secondaryDomains = {};
+            for (var env2 in $scope.archiveRestore.secondaryDomains) {
+                secondaryDomains[env2] = {
+                    subdomain: $scope.archiveRestore.secondaryDomains[env2].subdomain,
+                    domain: $scope.archiveRestore.secondaryDomains[env2].domain.domain
+                };
+            }
+
+            // only use enabled ports
+            var finalPorts = {};
+            for (var env in $scope.archiveRestore.ports) {
+                if ($scope.archiveRestore.portsEnabled[env]) {
+                    finalPorts[env] = $scope.archiveRestore.ports[env];
+                }
+            }
+
+            var data = {
+                subdomain: $scope.archiveRestore.subdomain,
+                domain: $scope.archiveRestore.domain.domain,
+                secondaryDomains: secondaryDomains,
+                ports: finalPorts,
+                overwriteDns: $scope.archiveRestore.overwriteDns,
+            };
+
+            var allDomains = [{ domain: data.domain, subdomain: data.subdomain }].concat(Object.keys(secondaryDomains).map(function (k) {
+                return {
+                    domain: secondaryDomains[k].domain,
+                    subdomain: secondaryDomains[k].subdomain
+                };
+            }));
+            async.eachSeries(allDomains, function (domain, callback) {
+                if ($scope.archiveRestore.overwriteDns) return callback();
+
+                Client.checkDNSRecords(domain.domain, domain.subdomain, function (error, result) {
+                    if (error) return callback(error);
+
+                    var fqdn = domain.subdomain + '.' + domain.domain;
+
+                    if (result.error) {
+                        if (result.error.reason === ERROR.ACCESS_DENIED) return callback({ type: 'provider', fqdn: fqdn, message: 'DNS credentials for ' + domain.domain + ' are invalid. Update it in Domains & Certs view' });
+                        return callback({ type: 'provider', fqdn: fqdn, message: result.error.message });
+                    }
+                    if (result.needsOverwrite) {
+                        $scope.archiveRestore.needsOverwrite = true;
+                        $scope.archiveRestore.overwriteDns = true;
+                        return callback({ type: 'externally_exists', fqdn: fqdn, message: 'DNS Record already exists. Confirm that the domain is not in use for services external to Cloudron' });
+                    }
+
+                    callback();
+                });
+            }, function (error) {
+                if (error) {
+                    if (error.type) {
+                        $scope.archiveRestore.error.location = error;
+                        $scope.archiveRestore.busy = false;
+                    } else {
+                        Client.error(error);
+                    }
+
+                    $scope.archiveRestore.error.location = error;
+                    $scope.archiveRestore.busy = false;
+                    return;
+                }
+
+                Client.unarchiveApp($scope.archiveRestore.archive.id, data, function (error/*, newApp */) {
+                    $scope.archiveRestore.busy = false;
+
+                    if (error) {
+                        var errorMessage = error.message.toLowerCase();
+                        if (errorMessage.indexOf('port') !== -1) {
+                            $scope.archiveRestore.error.port = error.message;
+                        } else if (error.message.indexOf('location') !== -1 || error.message.indexOf('subdomain') !== -1) {
+                            // TODO extract fqdn from error message, currently we just set it always to the main location
+                            $scope.archiveRestore.error.location = { type: 'internally_exists', fqdn: data.subdomain + '.' + data.domain, message: error.message };
+                            $('#cloneLocationInput').focus();
+                        } else {
+                            Client.error(error);
+                        }
+                        return;
+                    }
+
+                    $('#restoreArchiveModal').modal('hide');
+
+                    $location.path('/apps');
+                });
+            });
+        }
     };
 
     $scope.s3like = function (provider) {
@@ -296,7 +501,7 @@ angular.module('Application').controller('BackupsController', ['$scope', '$locat
         document.body.removeChild(element);
     }
 
-    $scope.downloadConfig = function (backup) {
+    $scope.downloadConfig = function (backup, isArchive) { // can also be a archive object
         // secrets and tokens already come with placeholder characters we remove them
         var tmp = {
             remotePath: backup.remotePath,
@@ -307,7 +512,13 @@ angular.module('Application').controller('BackupsController', ['$scope', '$locat
             if ($scope.backupConfig[k] !== SECRET_PLACEHOLDER) tmp[k] = $scope.backupConfig[k];
         });
 
-        var filename = 'cloudron-backup-config-' + (new Date()).toISOString().replace(/:|T/g,'-').replace(/\..*/,'') + ' (' + $scope.config.adminFqdn + ')' + '.json';
+        let filename;
+        if (isArchive) {
+            filename = `${backup.appConfig.fqdn}-archive-config-${(new Date(backup.creationTime)).toISOString().split('T')[0]}.json`;
+        } else {
+            filename = `${$scope.config.adminFqdn}-backup-config-${(new Date(backup.creationTime)).toISOString().split('T')[0]}.json`;
+        }
+
         download(filename, JSON.stringify(tmp, null, 4));
     };
 
@@ -860,12 +1071,15 @@ angular.module('Application').controller('BackupsController', ['$scope', '$locat
             fetchBackups();
             getBackupConfig();
 
+            $scope.archiveList.fetch();
+
             $scope.manualBackupApps = Client.getInstalledApps().filter(function (app) { return app.type !== APP_TYPES.LINK && !app.enableBackup; });
 
             // show backup status
             $scope.createBackup.init();
             $scope.cleanupBackups.init();
             $scope.backupPolicy.init();
+            $scope.archiveRestore.init();
 
             getBackupTasks();
             getCleanupTasks();

dashboard/public/views/domains.html

@@ -218,6 +218,17 @@
           </div>
 
           <p class="small text-info text-bold" ng-show="domainConfigure.provider === 'namecheap'" ng-bind-html="'domains.domainDialog.namecheapInfo' | tr"></p>
+
+          <!-- INWX -->
+          <div class="form-group" ng-class="{ 'has-error': false }" ng-show="domainConfigure.provider === 'inwx'">
+            <label class="control-label">{{ 'domains.domainDialog.inwxUsername' | tr }}</label>
+            <input type="text" class="form-control" ng-model="domainConfigure.inwxUsername" name="inwxUsername" ng-disabled="domainConfigure.busy" ng-required="domainConfigure.provider === 'inwx'">
+          </div>
+          <div class="form-group" ng-class="{ 'has-error': false }" ng-show="domainConfigure.provider === 'inwx'">
+            <label class="control-label">{{ 'domains.domainDialog.inwxPassword' | tr }}</label>
+            <input type="text" class="form-control" ng-model="domainConfigure.inwxPassword" name="inwxPassword" ng-disabled="domainConfigure.busy" ng-minlength="1" ng-required="domainConfigure.provider === 'inwx'">
+          </div>
+
           <p class="small text-info text-bold" ng-show="domainConfigure.provider === 'wildcard'" ng-bind-html="'domains.domainDialog.wildcardInfo' | tr:{ domain: domainConfigure.adding ? domainConfigure.newDomain : domainConfigure.domain.domain }"></p>
           <p class="small text-info text-bold" ng-show="domainConfigure.provider === 'manual'" ng-bind-html="'domains.domainDialog.manualInfo' | tr"></p>
           <p class="small text-info text-bold" ng-show="needsPort80(domainConfigure.provider, domainConfigure.tlsConfig.provider)" ng-bind-html="'domains.domainDialog.letsEncryptInfo' | tr"></p>

dashboard/public/views/domains.js

@@ -53,6 +53,7 @@ angular.module('Application').controller('DomainsController', ['$scope', '$locat
         { name: 'GoDaddy', value: 'godaddy' },
         { name: 'Google Cloud DNS', value: 'gcdns' },
         { name: 'Hetzner', value: 'hetzner' },
+        { name: 'INWX', value: 'inwx' },
         { name: 'Linode', value: 'linode' },
         { name: 'Name.com', value: 'namecom' },
         { name: 'Namecheap', value: 'namecheap' },
@@ -75,6 +76,7 @@ angular.module('Application').controller('DomainsController', ['$scope', '$locat
         case 'dnsimple': return 'dnsimple';
         case 'gandi': return 'Gandi LiveDNS';
         case 'hetzner': return 'Hetzner DNS';
+        case 'inwx': return 'INWX';
         case 'linode': return 'Linode';
         case 'namecom': return 'Name.com';
         case 'namecheap': return 'Namecheap';
@@ -275,6 +277,8 @@ angular.module('Application').controller('DomainsController', ['$scope', '$locat
         ovhAppSecret: '',
         porkbunSecretapikey: '',
         porkbunApikey: '',
+        inwxUsername: '',
+        inwxPassword: '',
 
         provider: 'route53',
         zoneName: '',
@@ -343,6 +347,9 @@ angular.module('Application').controller('DomainsController', ['$scope', '$locat
                 $scope.domainConfigure.namecheapApiKey = domain.provider === 'namecheap' ? domain.config.token : '';
                 $scope.domainConfigure.namecheapUsername = domain.provider === 'namecheap' ? domain.config.username : '';
 
+                $scope.domainConfigure.inwxUsername = domain.provider === 'inwx' ? domain.config.username : '';
+                $scope.domainConfigure.inwxPassword = domain.provider === 'inwx' ? domain.config.password : '';
+
                 $scope.domainConfigure.netcupCustomerNumber = domain.provider === 'netcup' ? domain.config.customerNumber : '';
                 $scope.domainConfigure.netcupApiKey = domain.provider === 'netcup' ? domain.config.apiKey : '';
                 $scope.domainConfigure.netcupApiPassword = domain.provider === 'netcup' ? domain.config.apiPassword : '';
@@ -428,6 +435,9 @@ angular.module('Application').controller('DomainsController', ['$scope', '$locat
             } else if (provider === 'namecheap') {
                 data.token = $scope.domainConfigure.namecheapApiKey;
                 data.username = $scope.domainConfigure.namecheapUsername;
+            } else if (provider === 'inwx') {
+                data.username = $scope.domainConfigure.inwxUsername;
+                data.password = $scope.domainConfigure.inwxPassword;
             } else if (provider === 'netcup') {
                 data.customerNumber = $scope.domainConfigure.netcupCustomerNumber;
                 data.apiKey = $scope.domainConfigure.netcupApiKey;
@@ -504,6 +514,8 @@ angular.module('Application').controller('DomainsController', ['$scope', '$locat
             $scope.domainConfigure.nameComUsername = '';
             $scope.domainConfigure.namecheapApiKey = '';
             $scope.domainConfigure.namecheapUsername = '';
+            $scope.domainConfigure.inwxUsername = '';
+            $scope.domainConfigure.inwxPassword = '';
             $scope.domainConfigure.netcupCustomerNumber = '';
             $scope.domainConfigure.netcupApiKey = '';
             $scope.domainConfigure.netcupApiPassword = '';

dashboard/public/views/emails.html

@@ -43,7 +43,7 @@
 </div>
 
 <!-- Modal solr config -->
-<div class="modal fade" id="solrConfigModal" tabindex="-1" role="dialog">
+<div class="modal fade" id="ftsConfigModal" tabindex="-1" role="dialog">
   <div class="modal-dialog">
     <div class="modal-content">
       <div class="modal-header">
@@ -52,12 +52,12 @@
       <div class="modal-body">
         <p ng-bind-html=" 'emails.solrConfig.description' | tr "></p>
         <!-- only show this when user is trying to enable -->
-        <p class="has-error" ng-show="!solrConfig.currentConfig.enabled && !solrConfig.enoughMemory">{{ 'emails.solrConfig.notEnoughMemory' | tr }}</p>
+        <p class="has-error" ng-show="!ftsConfig.currentConfig.enabled && !ftsConfig.enoughMemory">{{ 'emails.solrConfig.notEnoughMemory' | tr }}</p>
       </div>
       <div class="modal-footer">
         <button type="button" class="btn btn-default" data-dismiss="modal">{{ 'main.dialog.close' | tr }}</button>
-        <button type="button" class="btn btn-success" ng-hide="solrConfig.currentConfig.enabled" ng-click="solrConfig.submit(true)" ng-disabled="(!solrConfig.currentConfig.enabled && !solrConfig.enoughMemory) || solrConfig.busy"><i class="fa fa-circle-notch fa-spin" ng-show="solrConfig.busy"></i> {{ 'main.enableAction' | tr }}</button>
-        <button type="button" class="btn btn-danger" ng-show="solrConfig.currentConfig.enabled" ng-click="solrConfig.submit(false)" ng-disabled="solrConfig.busy"><i class="fa fa-circle-notch fa-spin" ng-show="solrConfig.busy"></i> {{ 'main.disableAction' | tr }}</button>
+        <button type="button" class="btn btn-success" ng-hide="ftsConfig.currentConfig.enabled" ng-click="ftsConfig.submit(true)" ng-disabled="(!ftsConfig.currentConfig.enabled && !ftsConfig.enoughMemory) || ftsConfig.busy"><i class="fa fa-circle-notch fa-spin" ng-show="ftsConfig.busy"></i> {{ 'main.enableAction' | tr }}</button>
+        <button type="button" class="btn btn-danger" ng-show="ftsConfig.currentConfig.enabled" ng-click="ftsConfig.submit(false)" ng-disabled="ftsConfig.busy"><i class="fa fa-circle-notch fa-spin" ng-show="ftsConfig.busy"></i> {{ 'main.disableAction' | tr }}</button>
       </div>
     </div>
   </div>
@@ -336,17 +336,17 @@
       <div class="col-xs-6">
         <span class="text-muted">{{ 'emails.settings.solrFts' | tr }}</span>
       </div>
-      <div class="col-xs-6 text-right" ng-hide="solrConfig.currentConfig">
+      <div class="col-xs-6 text-right" ng-hide="ftsConfig.currentConfig">
         <i class="fa fa-circle-notch fa-spin"></i>
       </div>
-      <div class="col-xs-6 text-right" ng-show="solrConfig.currentConfig">
-        <span ng-show="solrConfig.currentConfig.enabled">
+      <div class="col-xs-6 text-right" ng-show="ftsConfig.currentConfig">
+        <span ng-show="ftsConfig.currentConfig.enabled">
           {{ 'emails.settings.solrEnabled' | tr }}
-          <span ng-show="solrConfig.running">/ {{ 'emails.settings.solrRunning' | tr }}</span>
-          <span ng-hide="solrConfig.running">/ {{ 'emails.settings.solrNotRunning' | tr }}</span>
+          <span ng-show="ftsConfig.running">/ {{ 'emails.settings.solrRunning' | tr }}</span>
+          <span ng-hide="ftsConfig.running">/ {{ 'emails.settings.solrNotRunning' | tr }}</span>
         </span>
-        <span ng-hide="solrConfig.currentConfig.enabled">{{ 'emails.settings.solrDisabled' | tr }}</span>
-        <a href="" ng-click="solrConfig.show()"><i class="fa fa-edit text-small"></i></a>
+        <span ng-hide="ftsConfig.currentConfig.enabled">{{ 'emails.settings.solrDisabled' | tr }}</span>
+        <a href="" ng-click="ftsConfig.show()"><i class="fa fa-edit text-small"></i></a>
       </div>
     </div>
   </div>

dashboard/public/views/emails.js

@@ -196,7 +196,7 @@ angular.module('Application').controller('EmailsController', ['$scope', '$locati
         }
     };
 
-    $scope.solrConfig = {
+    $scope.ftsConfig = {
         busy: false,
         error: {},
         currentConfig: null, // null means not loaded yet
@@ -208,40 +208,40 @@ angular.module('Application').controller('EmailsController', ['$scope', '$locati
             Client.getService('mail', function (error, result) {
                 if (error) return console.log('Error getting status of mail conatiner', error);
 
-                $scope.solrConfig.enoughMemory = result.config.memoryLimit > (1024*1024*1024*2);
-                $scope.solrConfig.running = result.healthcheck && result.healthcheck.solr.status;
+                $scope.ftsConfig.enoughMemory = result.config.memoryLimit > (1024*1024*1024*2);
+                $scope.ftsConfig.running = result.healthcheck && result.healthcheck.solr.status && result.healthcheck.tika.status;
 
-                Client.getSolrConfig(function (error, config) {
+                Client.getFtsConfig(function (error, config) {
                     if (error) return console.error('Failed to get solr config', error);
 
-                    $scope.solrConfig.currentConfig = config;
+                    $scope.ftsConfig.currentConfig = config;
                 });
             });
         },
 
         show: function() {
-            $scope.solrConfig.busy = false;
-            $scope.solrConfig.error = null;
-            $scope.solrConfig.enabled = $scope.solrConfig.currentConfig.enabled;
+            $scope.ftsConfig.busy = false;
+            $scope.ftsConfig.error = null;
+            $scope.ftsConfig.enabled = $scope.ftsConfig.currentConfig.enabled;
 
-            $('#solrConfigModal').modal('show');
+            $('#ftsConfigModal').modal('show');
         },
 
         submit: function (newState) {
-            $scope.solrConfig.busy = true;
+            $scope.ftsConfig.busy = true;
 
-            Client.setSolrConfig(newState, function (error) {
+            Client.setFtsConfig(newState, function (error) {
                 if (error) return console.error(error);
 
                 $timeout(function () {
-                    $scope.solrConfig.busy = false;
+                    $scope.ftsConfig.busy = false;
                     // FIXME: these values are fake. but cannot get current status from mail server since it might be restarting
-                    $scope.solrConfig.currentConfig.enabled = newState;
-                    $scope.solrConfig.running = newState;
+                    $scope.ftsConfig.currentConfig.enabled = newState;
+                    $scope.ftsConfig.running = newState;
 
-                    $timeout(function () { $scope.solrConfig.refresh(); }, 20000); // get real values after 20 seconds
+                    $timeout(function () { $scope.ftsConfig.refresh(); }, 20000); // get real values after 20 seconds
 
-                    $('#solrConfigModal').modal('hide');
+                    $('#ftsConfigModal').modal('hide');
                 }, 5000);
             });
         }
@@ -502,7 +502,7 @@ angular.module('Application').controller('EmailsController', ['$scope', '$locati
                 $scope.virtualAllMail.refresh();
                 $scope.mailboxSharing.refresh();
                 $scope.spamConfig.refresh();
-                $scope.solrConfig.refresh();
+                $scope.ftsConfig.refresh();
                 $scope.acl.refresh();
             }
 

dashboard/public/views/eventlog.js

@@ -36,6 +36,9 @@ angular.module('Application').controller('EventLogController', ['$scope', '$loca
         { name: 'backup.cleanup.finish', value: 'backup.cleanup.finish' },
         { name: 'backup.finish', value: 'backup.finish' },
         { name: 'backup.start', value: 'backup.start' },
+        { name: 'branding.avatar', value: 'branding.avatar' },
+        { name: 'branding.footer', value: 'branding.footer' },
+        { name: 'branding.name', value: 'branding.name' },
         { name: 'certificate.new', value: 'certificate.new' },
         { name: 'certificate.renew', value: 'certificate.renew' },
         { name: 'certificate.cleanup', value: 'certificate.cleanup' },
@@ -52,6 +55,9 @@ angular.module('Application').controller('EventLogController', ['$scope', '$loca
         { name: 'domain.update', value: 'domain.update' },
         { name: 'domain.remove', value: 'domain.remove' },
         { name: 'externalldap.configure', value: 'externalldap.configure' },
+        { name: 'group.add', value: 'group.add' },
+        { name: 'group.update', value: 'group.update' },
+        { name: 'group.remove', value: 'group.remove' },
         { name: 'mail.location', value: 'mail.location' },
         { name: 'mail.enabled', value: 'mail.enabled' },
         { name: 'mail.box.add', value: 'mail.box.add' },

dashboard/public/views/notifications.html

@@ -1,8 +1,52 @@
+<!-- Modal configure notifications -->
+<div class="modal fade" id="notificationsSettingsModal" tabindex="-1" role="dialog">
+  <div class="modal-dialog">
+    <div class="modal-content">
+      <div class="modal-header">
+        <h4 class="modal-title">{{ 'notifications.settings.title' | tr:{ username: (userRemove.userInfo.username || userRemove.userInfo.email) } }}</h4>
+      </div>
+      <div class="modal-body">
+        <p>{{ 'notifications.settingsDialog.description' | tr }}</p>
+        <div class="checkbox">
+          <label>
+            <input type="checkbox" ng-model="settings.config.appUp"> {{ 'notifications.settings.appUp' | tr }}
+          </label>
+        </div>
+        <div class="checkbox">
+          <label>
+            <input type="checkbox" ng-model="settings.config.appDown"> {{ 'notifications.settings.appDown' | tr }}
+          </label>
+        </div>
+        <div class="checkbox">
+          <label>
+            <input type="checkbox" ng-model="settings.config.appOutOfMemory"> {{ 'notifications.settings.appOutOfMemory' | tr }}
+          </label>
+        </div>
+        <div class="checkbox">
+          <label>
+            <input type="checkbox" ng-model="settings.config.backupFailed"> {{ 'notifications.settings.backupFailed' | tr }}
+          </label>
+        </div>
+        <div class="checkbox">
+          <label>
+            <input type="checkbox" ng-model="settings.config.certificateRenewalFailed"> {{ 'notifications.settings.certificateRenewalFailed' | tr }}
+          </label>
+        </div>
+      </div>
+      <div class="modal-footer">
+        <button type="button" class="btn btn-default" data-dismiss="modal">{{ 'main.dialog.cancel' | tr }}</button>
+        <button type="button" class="btn btn-primary" ng-click="settings.submit()" ng-disabled="settings.busy"><i class="fa fa-circle-notch fa-spin" ng-show="settings.busy"></i> {{ 'main.saveAction' | tr }}</button>
+      </div>
+    </div>
+  </div>
+</div>
+
 <div class="content content-large">
 
   <h1 class="section-header">
     {{ 'notifications.title' | tr }}
     <div style="flex-grow: 1;"></div>
+    <button class="btn btn-default" ng-click="settings.show()"><i class="fas fa-mail-bulk"></i></button>
     <button class="btn btn-default" ng-click="showPrevPage()" ng-disabled="busy || currentPage <= 1"><i class="fa fa-angle-double-left"></i> {{ 'main.pagination.prev' | tr }}</button>
     <button class="btn btn-default" ng-click="showNextPage()" ng-disabled="busy || perPage > notifications.length">{{ 'main.pagination.next' | tr }} <i class="fa fa-angle-double-right"></i></button>
     <button class="btn btn-primary" ng-click="clearAll()" ng-disabled="!$parent.notificationCount || clearAllBusy"><i class="fa fa-circle-notch fa-spin" ng-show="clearAllBusy"></i><i class="fa fa-check" ng-hide="clearAllBusy"></i> {{ 'notifications.markAllAsRead' | tr }}</button>

dashboard/public/views/notifications.js

@@ -1,7 +1,7 @@
 'use strict';
 
 /* global async */
-/* global angular */
+/* global angular, $ */
 
 angular.module('Application').controller('NotificationsController', ['$scope', '$location', '$timeout', '$translate', '$interval', 'Client', function ($scope, $location, $timeout, $translate, $interval, Client) {
     Client.onReady(function () { if (!Client.getUserInfo().isAtLeastAdmin) $location.path('/'); });
@@ -80,6 +80,29 @@ angular.module('Application').controller('NotificationsController', ['$scope', '
         });
     };
 
+    $scope.settings = {
+        busy: false,
+        config: {},
+
+        show: function () {
+            for (const s of Client.getUserInfo().notificationConfig) {
+                $scope.settings.config[s] = true;
+            }
+
+            $('#notificationsSettingsModal').modal('show');
+        },
+
+        submit: function () {
+            const config = Object.keys($scope.settings.config).filter(c => $scope.settings.config[c] === true);
+            Client.setNotificationConfig(config, function (error) {
+                if (error) return Client.error(error);
+
+                Client.refreshProfile();
+                $('#notificationsSettingsModal').modal('hide');
+            });
+        },
+    };
+
     Client.onReady(function () {
         var refreshTimer = $interval($scope.refresh, 60 * 1000); // keep this interval in sync with the notification count indicator in main.js
         $scope.$on('$destroy', function () {

dashboard/public/views/services.js

@@ -86,9 +86,12 @@ angular.module('Application').controller('ServicesController', ['$scope', '$loca
             var nearest256m = Math.ceil(Math.max($scope.memory.memory, service.config.memoryLimit) / (256*1024*1024)) * 256 * 1024 * 1024;
             var startTick = service.defaultMemoryLimit;
 
-            for (var i = startTick; i <= nearest256m; i *= 2) {
+            // code below ensure we atleast have 2 ticks to keep the slider usable
+            $scope.serviceConfigure.memoryTicks.push(startTick); // start tick
+            for (var i = startTick * 2; i < nearest256m; i *= 2) {
                 $scope.serviceConfigure.memoryTicks.push(i);
             }
+            $scope.serviceConfigure.memoryTicks.push(nearest256m); // end tick
 
             // for firefox widget update
             $timeout(function() {

dashboard/public/views/system.js

@@ -74,7 +74,7 @@ angular.module('Application').controller('SystemController', ['$scope', '$locati
                 $scope.disks.ts = result.usage.ts;
 
                 // [ { filesystem, type, size, used, available, capacity, mountpoint }]
-                $scope.disks.disks = Object.keys(result.usage.disks).map(function (k) { return result.usage.disks[k]; });
+                $scope.disks.disks = Object.keys(result.usage.filesystems).map(function (k) { return result.usage.filesystems[k]; }); // convert object to array...
 
                 $scope.disks.disks.forEach(function (disk) {
                     var usageOther = disk.used;

dashboard/public/views/user-directory.html

@@ -215,7 +215,7 @@
   <div class="modal-dialog">
     <div class="modal-content">
       <div class="modal-header">
-        <h4 class="modal-title">{{ 'oidc.deleteClientDialog.title' | tr:{ client: deleteClient.id } }}</h4>
+        <h4 class="modal-title">{{ 'oidc.deleteClientDialog.title' | tr:{ client: deleteClient.name } }}</h4>
       </div>
       <div class="modal-body">
         <p>{{ 'oidc.deleteClientDialog.description' | tr }}</p>

dashboard/public/views/user-directory.js

@@ -388,10 +388,12 @@ angular.module('Application').controller('UserSettingsController', ['$scope', '$
         busy: false,
         error: {},
         id: '',
+        name: '',
 
         show: function (client) {
             $scope.deleteClient.busy = false;
             $scope.deleteClient.id = client.id;
+            $scope.deleteClient.name = client.name;
 
             $('#oidcClientDeleteModal').modal('show');
         },

dashboard/public/views/users.html

@@ -382,7 +382,7 @@
             <div class="input-group">
               <input type="text" id="setGhostPassword" class="form-control" name="ghostPassword" ng-model="setGhost.password" required ng-readonly="setGhost.success"/>
               <span class="input-group-btn">
-                <button class="btn btn-default" ng-hide="setGhost.success" type="button" uib-tooltip="{{ 'users.setGhostDialog.generatePassword' | tr }}Generate Password" ng-click="setGhost.generatePassword()"><i class="fa fa-key"></i></button>
+                <button class="btn btn-default" ng-hide="setGhost.success" type="button" uib-tooltip="{{ 'users.setGhostDialog.generatePassword' | tr }}" ng-click="setGhost.generatePassword()"><i class="fa fa-key"></i></button>
                 <button class="btn btn-default" ng-show="setGhost.success" type="button" id="setGhostClipboardButton" data-clipboard-target="#setGhostPassword"><i class="fa fa-clipboard"></i></button>
               </span>
             </div>

dashboard/setup.html

@@ -222,6 +222,16 @@
                   <input type="text" class="form-control" ng-model="dnsCredentials.namecheapApiKey" name="namecheapApiKey" placeholder="Namecheap API Key" ng-required="dnsCredentials.provider === 'namecheap'" ng-disabled="dnsCredentials.busy">
                 </div>
 
+                <!-- INWX -->
+                <div class="form-group" ng-class="{ 'has-error': dnsCredentialsForm.inwxUsername.$dirty && dnsCredentialsForm.inwxUsername.$invalid }" ng-show="dnsCredentials.provider === 'inwx'">
+                  <label class="control-label">INWX Username</label>
+                  <input type="text" class="form-control" ng-model="dnsCredentials.inwxUsername" name="inwxUsername" placeholder="INWX Username" ng-required="dnsCredentials.provider === 'inwx'" ng-disabled="dnsCredentials.busy">
+                </div>
+                <div class="form-group" ng-class="{ 'has-error': dnsCredentialsForm.inwxPassword.$dirty && dnsCredentialsForm.inwxPassword.$invalid }" ng-show="dnsCredentials.provider === 'inwx'">
+                  <label class="control-label">INWX Password</label>
+                  <input type="text" class="form-control" ng-model="dnsCredentials.inwxPassword" name="inwxPassword" placeholder="INWX Password" ng-required="dnsCredentials.provider === 'inwx'" ng-disabled="dnsCredentials.busy">
+                </div>
+
                 <!-- Linode -->
                 <p class="form-group" ng-show="dnsCredentials.provider === 'linode'">
                   <label class="control-label">API Token</label>

dashboard/src/models/DirectoryModel.js

@@ -187,7 +187,7 @@ export function createDirectoryModel(origin, accessToken, api) {
           if (action === 'copy') result = await this.copy(this.buildFilePath(files[f].folderPath, files[f].name), targetPath);
 
           if (result.status === 404) {
-            throw `Source file ${files[f].name} not found`;
+            throw `Source file "${files[f].name}" not found`;
           } else if (result.status === 409) {
             targetPath += '-copy';
             continue;

dashboard/src/modules.js

@@ -6,7 +6,7 @@ import '@fontsource/noto-sans';
 import 'bootstrap-sass';
 import Chart from 'chart.js/auto';
 
-import * as moment from 'moment/dist/moment.js';
+import * as moment from 'moment/min/moment-with-locales';
 
 // attach to global for compatibility
 window.moment = moment.default;

dashboard/src/theme.scss

@@ -1665,6 +1665,12 @@ div:hover > .picture-edit-indicator {
     overflow: hidden;
 }
 
+.no-wrap-scroll {
+    text-wrap: nowrap;
+    overflow: auto;
+    display: block;
+}
+
 .users-toolbar {
     display: flex;
     margin-bottom: 5px;

migrations/20241205131851-apps-add-devicesJson.js

@@ -0,0 +1,15 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps ADD COLUMN devicesJson TEXT', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN devicesJson', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20241206163257-locks-create-table.js

@@ -0,0 +1,18 @@
+'use strict';
+
+'use strict';
+
+exports.up = async function (db) {
+    const cmd = 'CREATE TABLE IF NOT EXISTS locks(' +
+        'id VARCHAR(128) NOT NULL UNIQUE,' +
+        'dataJson TEXT,' +
+        'version INT DEFAULT 1,' +
+        'ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP' +
+        ') CHARACTER SET utf8 COLLATE utf8_bin';
+
+    await db.runSql(cmd);
+};
+
+exports.down = async function (db) {
+    await db.runSql('DROP TABLE locks');
+};

migrations/20241209150823-archives-add-table.js

@@ -0,0 +1,19 @@
+'use strict';
+
+exports.up = async function (db) {
+    const cmd = 'CREATE TABLE archives(' +
+        'id VARCHAR(128) NOT NULL UNIQUE,' +
+        'backupId VARCHAR(128) NOT NULL,' +
+        'creationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,' +
+        'appStoreIcon MEDIUMBLOB,' +
+        'icon MEDIUMBLOB,' +
+        'FOREIGN KEY(backupId) REFERENCES backups(id),' +
+        'PRIMARY KEY (id)) ' +
+        'CHARACTER SET utf8 COLLATE utf8_bin';
+
+    await db.runSql(cmd);
+};
+
+exports.down = async function (db) {
+    await db.runSql('DROP TABLE archives');
+};

migrations/20241210192343-backups-add-appConfig.js

@@ -0,0 +1,9 @@
+'use strict';
+
+exports.up = async function (db) {
+    await db.runSql('ALTER TABLE backups ADD COLUMN appConfigJson TEXT');
+};
+
+exports.down = async function (db) {
+    await db.runSql('ALTER TABLE backups DROP COLUMN appConfigJson');
+};

migrations/20241211175838-users-add-notificationConfig.js

@@ -0,0 +1,9 @@
+'use strict';
+
+exports.up = async function (db) {
+    await db.runSql('ALTER TABLE users ADD COLUMN notificationConfigJson TEXT');
+};
+
+exports.down = async function (db) {
+    await db.runSql('ALTER TABLE users DROP COLUMN notificationConfigJson');
+};

migrations/20241211205706-notifications-add-context.js

@@ -0,0 +1,9 @@
+'use strict';
+
+exports.up = async function (db) {
+    await db.runSql('ALTER TABLE notifications ADD COLUMN context VARCHAR(128) DEFAULT ""');
+};
+
+exports.down = async function (db) {
+    await db.runSql('ALTER TABLE notifications DROP COLUMN context');
+};

migrations/schema.sql

@@ -35,6 +35,7 @@ CREATE TABLE IF NOT EXISTS users(
     avatar MEDIUMBLOB NOT NULL,
     backgroundImage MEDIUMBLOB,
     loginLocationsJson MEDIUMTEXT, // { locations: [{ ip, userAgent, city, country, ts }] }
+    notificationConfigJson TEXT,
 
     INDEX creationTime_index (creationTime),
     PRIMARY KEY(id));
@@ -80,6 +81,7 @@ CREATE TABLE IF NOT EXISTS apps(
     cpuQuota INTEGER DEFAULT 100,
     xFrameOptions VARCHAR(512),
     sso BOOLEAN DEFAULT 1, // whether user chose to enable SSO
+    devicesJson TEXT,
     debugModeJson TEXT, // options for development mode
     reverseProxyConfigJson TEXT, // { robotsTxt, csp, hstsPreload }
     enableBackup BOOLEAN DEFAULT 1, // misnomer: controls automatic daily backups
@@ -155,10 +157,21 @@ CREATE TABLE IF NOT EXISTS backups(
     manifestJson TEXT, /* to validate if the app can be installed in this version of box */
     format VARCHAR(16) DEFAULT "tgz",
     preserveSecs INTEGER DEFAULT 0,
+    appConfigJson TEXT, /* useful for clone and archive */
 
     INDEX creationTime_index (creationTime),
     PRIMARY KEY (id));
 
+CREATE TABLE IF NOT EXISTS archives(
+    id VARCHAR(128) NOT NULL UNIQUE,
+    backupId VARCHAR(128) NOT NULL,
+    creationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    appStoreIcon MEDIUMBLOB,
+    icon MEDIUMBLOB,
+
+    FOREIGN KEY(backupId) REFERENCES backups(id),
+    PRIMARY KEY (id));
+
 CREATE TABLE IF NOT EXISTS eventlog(
     id VARCHAR(128) NOT NULL,
     action VARCHAR(128) NOT NULL,
@@ -201,11 +214,7 @@ CREATE TABLE IF NOT EXISTS mail(
 
     CHARACTER SET utf8 COLLATE utf8_bin;
 
-/* Future fields:
-   * accessRestriction - to determine who can access it. So this has foreign keys
-   * quota - per mailbox quota
-
-   NOTE: this table exists only real mailboxes. And has unique constraint to handle
+/* NOTE: this table contains only real mailboxes. And has unique constraint to handle
    conflict with aliases and mailbox names
 */
 CREATE TABLE IF NOT EXISTS mailboxes(
@@ -263,6 +272,7 @@ CREATE TABLE IF NOT EXISTS notifications(
     message TEXT,
     acknowledged BOOLEAN DEFAULT false,
     creationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    context VARCHAR(128) DEFAULT "", // used along with "type" to create uniqueness
 
     INDEX creationTime_index (creationTime),
     FOREIGN KEY(eventId) REFERENCES eventlog(id),
@@ -326,3 +336,10 @@ CREATE TABLE IF NOT EXISTS oidcClients(
     loginRedirectUri VARCHAR(256) DEFAULT "",
     tokenSignatureAlgorithm VARCHAR(128) DEFAULT "",
     PRIMARY KEY(id));
+
+CREATE TABLE IF NOT EXISTS locks(
+    id VARCHAR(128) NOT NULL UNIQUE,
+    dataJson TEXT,
+    version INT DEFAULT 1
+    ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP);
+

package-lock.json

@@ -13,7 +13,7 @@
         "async": "^3.2.5",
         "aws-sdk": "^2.1637.0",
         "basic-auth": "^2.0.1",
-        "cloudron-manifestformat": "^5.24.0",
+        "cloudron-manifestformat": "^5.26.2",
         "connect": "^3.7.0",
         "connect-lastmile": "^2.2.0",
         "connect-timeout": "^1.9.0",
@@ -24,6 +24,7 @@
         "db-migrate-mysql": "^2.3.2",
         "debug": "^4.3.5",
         "dockerode": "^4.0.2",
+        "domrobot-client": "^3.2.2",
         "ejs": "^3.1.10",
         "express": "^4.19.2",
         "ipaddr.js": "^2.2.0",
@@ -420,6 +421,53 @@
         "node": ">= 8"
       }
     },
+    "node_modules/@otplib/core": {
+      "version": "12.0.1",
+      "resolved": "https://registry.npmjs.org/@otplib/core/-/core-12.0.1.tgz",
+      "integrity": "sha512-4sGntwbA/AC+SbPhbsziRiD+jNDdIzsZ3JUyfZwjtKyc/wufl1pnSIaG4Uqx8ymPagujub0o92kgBnB89cuAMA==",
+      "license": "MIT"
+    },
+    "node_modules/@otplib/plugin-crypto": {
+      "version": "12.0.1",
+      "resolved": "https://registry.npmjs.org/@otplib/plugin-crypto/-/plugin-crypto-12.0.1.tgz",
+      "integrity": "sha512-qPuhN3QrT7ZZLcLCyKOSNhuijUi9G5guMRVrxq63r9YNOxxQjPm59gVxLM+7xGnHnM6cimY57tuKsjK7y9LM1g==",
+      "license": "MIT",
+      "dependencies": {
+        "@otplib/core": "^12.0.1"
+      }
+    },
+    "node_modules/@otplib/plugin-thirty-two": {
+      "version": "12.0.1",
+      "resolved": "https://registry.npmjs.org/@otplib/plugin-thirty-two/-/plugin-thirty-two-12.0.1.tgz",
+      "integrity": "sha512-MtT+uqRso909UkbrrYpJ6XFjj9D+x2Py7KjTO9JDPhL0bJUYVu5kFP4TFZW4NFAywrAtFRxOVY261u0qwb93gA==",
+      "license": "MIT",
+      "dependencies": {
+        "@otplib/core": "^12.0.1",
+        "thirty-two": "^1.0.2"
+      }
+    },
+    "node_modules/@otplib/preset-default": {
+      "version": "12.0.1",
+      "resolved": "https://registry.npmjs.org/@otplib/preset-default/-/preset-default-12.0.1.tgz",
+      "integrity": "sha512-xf1v9oOJRyXfluBhMdpOkr+bsE+Irt+0D5uHtvg6x1eosfmHCsCC6ej/m7FXiWqdo0+ZUI6xSKDhJwc8yfiOPQ==",
+      "license": "MIT",
+      "dependencies": {
+        "@otplib/core": "^12.0.1",
+        "@otplib/plugin-crypto": "^12.0.1",
+        "@otplib/plugin-thirty-two": "^12.0.1"
+      }
+    },
+    "node_modules/@otplib/preset-v11": {
+      "version": "12.0.1",
+      "resolved": "https://registry.npmjs.org/@otplib/preset-v11/-/preset-v11-12.0.1.tgz",
+      "integrity": "sha512-9hSetMI7ECqbFiKICrNa4w70deTUfArtwXykPUvSHWOdzOlfa9ajglu7mNCntlvxycTiOAXkQGwjQCzzDEMRMg==",
+      "license": "MIT",
+      "dependencies": {
+        "@otplib/core": "^12.0.1",
+        "@otplib/plugin-crypto": "^12.0.1",
+        "@otplib/plugin-thirty-two": "^12.0.1"
+      }
+    },
     "node_modules/@sindresorhus/is": {
       "version": "5.6.0",
       "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-5.6.0.tgz",
@@ -1086,13 +1134,14 @@
       }
     },
     "node_modules/cloudron-manifestformat": {
-      "version": "5.24.0",
-      "resolved": "https://registry.npmjs.org/cloudron-manifestformat/-/cloudron-manifestformat-5.24.0.tgz",
-      "integrity": "sha512-thmYEX9EFfpxnla/MQvo0GX9Hwr26LDbJKA9DVS1vIWP34M+LWza9PVhDaYii1sDKW4hH8C3hwFeqjrfWw8UBw==",
+      "version": "5.26.2",
+      "resolved": "https://registry.npmjs.org/cloudron-manifestformat/-/cloudron-manifestformat-5.26.2.tgz",
+      "integrity": "sha512-o8q4HXQvYHIbZK4xftzffO2gTzw+U0797IiXzdVNszpEUkcoqeN1RNvRdAYl3vXPE92VL3MF0o5iN8tBFoWbng==",
+      "license": "MIT",
       "dependencies": {
-        "cron": "^3.1.7",
+        "cron": "^3.2.1",
         "safetydance": "2.4.0",
-        "semver": "^7.6.2",
+        "semver": "^7.6.3",
         "tv4": "^1.3.0",
         "validator": "^13.12.0"
       }
@@ -1362,12 +1411,13 @@
       }
     },
     "node_modules/cron": {
-      "version": "3.1.7",
-      "resolved": "https://registry.npmjs.org/cron/-/cron-3.1.7.tgz",
-      "integrity": "sha512-tlBg7ARsAMQLzgwqVxy8AZl/qlTc5nibqYwtNGoCrd+cV+ugI+tvZC1oT/8dFH8W455YrywGykx/KMmAqOr7Jw==",
+      "version": "3.2.1",
+      "resolved": "https://registry.npmjs.org/cron/-/cron-3.2.1.tgz",
+      "integrity": "sha512-w2n5l49GMmmkBFEsH9FIDhjZ1n1QgTMOCMGuQtOXs5veNiosZmso6bQGuqOJSYAXXrG84WQFVneNk+Yt0Ua9iw==",
+      "license": "MIT",
       "dependencies": {
         "@types/luxon": "~3.4.0",
-        "luxon": "~3.4.0"
+        "luxon": "~3.5.0"
       }
     },
     "node_modules/cross-spawn": {
@@ -1761,6 +1811,17 @@
         "node": ">=6"
       }
     },
+    "node_modules/domrobot-client": {
+      "version": "3.2.2",
+      "resolved": "https://registry.npmjs.org/domrobot-client/-/domrobot-client-3.2.2.tgz",
+      "integrity": "sha512-9q9uVOYi/4K0Sa0JcLIf+AX0Xsh/1OTSyaBXuB4l2LEPacoFb/CWMMw76OSLkV5kyDJ+Igb1NRjQpdRsVr0qlg==",
+      "dependencies": {
+        "otplib": "^12.0.1"
+      },
+      "engines": {
+        "node": ">=18"
+      }
+    },
     "node_modules/dotenv": {
       "version": "5.0.1",
       "license": "BSD-2-Clause",
@@ -3621,9 +3682,10 @@
       }
     },
     "node_modules/luxon": {
-      "version": "3.4.4",
-      "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.4.4.tgz",
-      "integrity": "sha512-zobTr7akeGHnv7eBOXcRgMeCP6+uyYsczwmeRCauvpvaAltgNyTbLH/+VaEAPUeWBT+1GuNmz4wC/6jtQzbbVA==",
+      "version": "3.5.0",
+      "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.5.0.tgz",
+      "integrity": "sha512-rh+Zjr6DNfUYR3bPwJEnuwDdqMbxZW7LOQfUN4B54+Cl+0o5zaU9RJ6bcidfDtC1cWCZXQ+nvX8bf6bAji37QQ==",
+      "license": "MIT",
       "engines": {
         "node": ">=12"
       }
@@ -4310,6 +4372,17 @@
         "node": ">= 0.8.0"
       }
     },
+    "node_modules/otplib": {
+      "version": "12.0.1",
+      "resolved": "https://registry.npmjs.org/otplib/-/otplib-12.0.1.tgz",
+      "integrity": "sha512-xDGvUOQjop7RDgxTQ+o4pOol0/3xSZzawTiPKRrHnQWAy0WjhNs/5HdIDJCrqC4MBynmjXgULc6YfioaxZeFgg==",
+      "license": "MIT",
+      "dependencies": {
+        "@otplib/core": "^12.0.1",
+        "@otplib/preset-default": "^12.0.1",
+        "@otplib/preset-v11": "^12.0.1"
+      }
+    },
     "node_modules/ovh": {
       "version": "2.0.3",
       "resolved": "https://registry.npmjs.org/ovh/-/ovh-2.0.3.tgz",
@@ -4923,9 +4996,10 @@
       }
     },
     "node_modules/semver": {
-      "version": "7.6.2",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.2.tgz",
-      "integrity": "sha512-FNAIBWCx9qcRhoHcgcJ0gvU7SN1lYU2ZXuSfl04bSC5OpvDHFyJCjdNHomPXxjQlCBU67YW64PzY7/VIEH7F2w==",
+      "version": "7.6.3",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz",
+      "integrity": "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==",
+      "license": "ISC",
       "bin": {
         "semver": "bin/semver.js"
       },
@@ -5339,6 +5413,14 @@
       "integrity": "sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==",
       "dev": true
     },
+    "node_modules/thirty-two": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/thirty-two/-/thirty-two-1.0.2.tgz",
+      "integrity": "sha512-OEI0IWCe+Dw46019YLl6V10Us5bi574EvlJEOcAkB29IzQ/mYD1A6RyNHLjZPiHCmuodxvgF6U+vZO1L15lxVA==",
+      "engines": {
+        "node": ">=0.2.6"
+      }
+    },
     "node_modules/tldjs": {
       "version": "2.3.1",
       "resolved": "https://registry.npmjs.org/tldjs/-/tldjs-2.3.1.tgz",

package.json

@@ -21,7 +21,7 @@
     "async": "^3.2.5",
     "aws-sdk": "^2.1637.0",
     "basic-auth": "^2.0.1",
-    "cloudron-manifestformat": "^5.24.0",
+    "cloudron-manifestformat": "^5.26.2",
     "connect": "^3.7.0",
     "connect-lastmile": "^2.2.0",
     "connect-timeout": "^1.9.0",
@@ -32,6 +32,7 @@
     "db-migrate-mysql": "^2.3.2",
     "debug": "^4.3.5",
     "dockerode": "^4.0.2",
+    "domrobot-client": "^3.2.2",
     "ejs": "^3.1.10",
     "express": "^4.19.2",
     "ipaddr.js": "^2.2.0",

run-tests

@@ -23,7 +23,7 @@ mkdir -p ${DATA_DIR}
 cd ${DATA_DIR}
 mkdir -p appsdata
 mkdir -p boxdata/box boxdata/mail boxdata/certs boxdata/mail/dkim/localhost boxdata/mail/dkim/foobar.com
-mkdir -p platformdata/addons/mail/banner platformdata/nginx/cert platformdata/nginx/applications/dashboard platformdata/collectd/collectd.conf.d platformdata/addons platformdata/logrotate.d platformdata/backup platformdata/logs/tasks platformdata/sftp/ssh platformdata/firewall platformdata/update
+mkdir -p platformdata/addons/mail/banner platformdata/nginx/cert platformdata/nginx/applications/dashboard platformdata/collectd/collectd.conf.d platformdata/addons platformdata/logrotate.d platformdata/backup platformdata/logs/tasks platformdata/sftp/ssh platformdata/firewall platformdata/update platformdata/diskusage
 sudo mkdir -p /mnt/cloudron-test-music /media/cloudron-test-music # volume test
 
 # put cert

scripts/cloudron-support

@@ -354,6 +354,7 @@ function check_node() {
         echo "You can try the following to fix the problem:"
         echo "    ln -sf /usr/local/node-${expected_node_version}/bin/node /usr/bin/node"
         echo "    ln -sf /usr/local/node-${expected_node_version}/bin/npm /usr/bin/npm"
+        echo "    apt remove -y nodejs"
         echo "    systemctl restart box"
         exit 1
     fi
@@ -390,12 +391,12 @@ function check_ipv6() {
     done
 
     if [[ "${has_ipv6_address}" == "0" ]]; then
-        success "IPv6 is enabled. No public IPv6 address"
+        success "IPv6 is enabled in kernel. No public IPv6 address"
         return
     fi
 
-    if ! ping6 -q -c 1 api.cloudron.io; then
-        fail "Server has an IPv6 address but api.cloudron.io is unreachable via IPv6"
+    if ! ping6 -q -c 1 api.cloudron.io >/dev/null 2>&1; then
+        fail "Server has an IPv6 address but api.cloudron.io is unreachable via IPv6 (ping6 -q -c 1 api.cloudron.io)"
         print_ipv6_disable_howto
         exit 1
     fi
@@ -463,8 +464,8 @@ function check_dashboard_site_domain() {
     local -r dashboard_domain=$(mysql -NB -uroot -ppassword -e "SELECT value FROM box.settings WHERE name='dashboard_domain'" 2>/dev/null)
     local -r domain_provider=$(mysql -NB -uroot -ppassword -e "SELECT provider FROM box.domains WHERE domain='${dashboard_domain}'" 2>/dev/null)
 
-    # TODO: check ipv4 and ipv6
-    if ! output=$(curl --fail -s https://my.${dashboard_domain}); then
+    # TODO: check ipv4 and ipv6 separately
+    if ! output=$(curl --fail --connect-timeout 10 --max-time 20 -s https://my.${dashboard_domain}); then
         fail "Could not load dashboard domain."
         if [[ "${domain_provider}" == "cloudflare" ]]; then
             echo "Maybe cloudflare proxying is not working. Delete the domain in Cloudflare dashboard and re-add it. This sometimes re-establishes the proxying"

scripts/installer.sh

@@ -105,8 +105,9 @@ if dpkg -s resolvconf 2>/dev/null >/dev/null; then
 fi
 
 # https://docs.docker.com/engine/installation/linux/ubuntulinux/
-readonly docker_version="26.0.1"
-readonly containerd_version="1.6.31-1"
+# https://download.docker.com/linux/ubuntu/dists/noble/pool/stable/amd64/
+readonly docker_version="27.3.1"
+readonly containerd_version="1.7.23"
 if ! which docker 2>/dev/null || [[ $(docker version --format {{.Client.Version}}) != "${docker_version}" ]]; then
     log "installing/updating docker"
 
@@ -115,7 +116,7 @@ if ! which docker 2>/dev/null || [[ $(docker version --format {{.Client.Version}
     echo -e "[Service]\nExecStart=\nExecStart=/usr/bin/dockerd -H fd:// --log-driver=journald --exec-opt native.cgroupdriver=cgroupfs --storage-driver=overlay2 --experimental --ip6tables" > /etc/systemd/system/docker.service.d/cloudron.conf
 
     # there are 3 packages for docker - containerd, CLI and the daemon (https://download.docker.com/linux/ubuntu/dists/jammy/pool/stable/amd64/)
-    $curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/containerd.io_${containerd_version}_amd64.deb" -o /tmp/containerd.deb
+    $curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/containerd.io_${containerd_version}-1_amd64.deb" -o /tmp/containerd.deb
     $curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/docker-ce-cli_${docker_version}-1~ubuntu.${ubuntu_version}~${ubuntu_codename}_amd64.deb" -o /tmp/docker-ce-cli.deb
     $curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/docker-ce_${docker_version}-1~ubuntu.${ubuntu_version}~${ubuntu_codename}_amd64.deb" -o /tmp/docker.deb
 
@@ -158,18 +159,33 @@ fi
 log "downloading new addon images"
 images=$(node -e "const i = require('${box_src_tmp_dir}/src/infra_version.js'); console.log(Object.keys(i.images).map(x => i.images[x]).join(' '));")
 
+# docker hub only uses first 64 bits for ipv6 addressing. this causes many ipv6 rate limit errors
+# https://www.docker.com/blog/beta-ipv6-support-on-docker-hub-registry/
 log "\tPulling docker images: ${images}"
-for image in ${images}; do
-    while ! docker pull "${image}"; do           # this pulls the image using the sha256
-        log "Could not pull ${image}"
-        sleep 5
+for image_ref in ${images}; do
+    ipv4_image_ref="${image_ref/registry.docker.com/registry.ipv4.docker.com}"
+    ipv6_image_ref="${image_ref/registry.docker.com/registry.ipv6.docker.com}"
+
+    while true; do
+        if docker pull "${ipv4_image_ref}"; then # this pulls the image untagged using the sha256 but doesn't tag it!
+            docker tag "${ipv4_image_ref}" "${image_ref%@sha256:*}" # this will tag the image for readability
+            docker rmi "${ipv4_image_ref}"
+            break
+        fi
+        log "Could not pull ${ipv4_image_ref} , trying IPv6"
+        if docker pull "${ipv6_image_ref}"; then # this pulls the image untagged using the sha256 but doesn't tag it!
+            docker tag "${ipv6_image_ref}" "${image_ref%@sha256:*}" # this will tag the image for readability
+            docker rmi "${ipv6_image_ref}"
+            break
+        fi
+        log "Could not pull ${ipv6_image_ref} either, waiting for 10s"
+        sleep 10
     done
-    while ! docker pull "${image%@sha256:*}"; do  # this will tag the image for readability
-        log "Could not pull ${image%@sha256:*}"
-        sleep 5
-   done
 done
 
+# remove after 8.2 . we used to have infra.base and this tag will prevent clean up
+docker rmi -f registry.docker.com/cloudron/base:4.2.0 >/dev/null 2>&1 || true
+
 if [[ "${is_update}" == "yes" ]]; then
     log "stop box service for update"
     ${box_src_dir}/setup/stop.sh

scripts/release

@@ -5,6 +5,7 @@
 const assert = require('assert'),
     { execSync, spawnSync } = require('child_process'),
     fs = require('fs'),
+    net = require('net'),
     os = require('os'),
     path = require('path'),
     { program } = require('commander'),
@@ -38,11 +39,7 @@ const ENVIRONMENTS = {
 function exit(error) {
     if (error) console.error(error.message);
 
-    // we don't call process.exit() immediately, as it does not wait for the async console. api to print remaining logs
-    // this is ugly but effective until we find a better way to flush console first
-    setTimeout(function () {
-        process.exit(error ? 1 : 0);
-    }, 250);
+    process.exit(error ? 1 : 0);
 }
 
 function parseChangelog(version) {
@@ -125,20 +122,20 @@ async function uploadVersionsJSON(env, releases) {
     assert.strictEqual(typeof env, 'object');
     assert.strictEqual(typeof releases, 'object');
 
-    console.log('Computing GPG signature of versions.json...');
+    console.log(`[${env.tag}] Computing GPG signature of versions.json...`);
     await fs.promises.writeFile('/tmp/versions.json', JSON.stringify(releases, null, 4));
     await fs.promises.rm('/tmp/versions.json.sig', { force: true });
 
     execSync('gpg --no-default-keyring --local-user 0EADB19CDDA23CD0FE71E3470A372F8703C493CC --output /tmp/versions.json.sig --detach-sig /tmp/versions.json',
         { stdio: [ null, process.stdout, process.stderr ] } );
 
-    console.log('Uploading versions.json');
+    console.log(`[${env.tag}] Uploading versions.json`);
     execSync(`rsync /tmp/versions.json ubuntu@${env.releasesServer}:/home/ubuntu/releases/`, { stdio: [ null, process.stdout, process.stderr ] } );
 
-    console.log('Uploading versions.json.sig');
+    console.log(`[${env.tag}] Uploading versions.json.sig`);
     execSync(`rsync /tmp/versions.json.sig ubuntu@${env.releasesServer}:/home/ubuntu/releases/`,         { stdio: [ null, process.stdout, process.stderr ] } );
 
-    console.log('versions.json and signature uploaded');
+    console.log(`[${env.tag}] versions.json and signature uploaded`);
 }
 
 async function verifyAndUpload(env, releases) {
@@ -205,15 +202,15 @@ async function createRelease(options) {
         if (!fs.existsSync(options.code)) return exit('code must be a valid file');
 
         // "gpgconf --reload gpg-agent" is handy to reset existing password in the agent. See https://dev.gnupg.org/T3485 for pinentry-mode (--pinentry-mode=loopback --batch --passphrase ${passphrase} works if we want to gassword protect
-        console.log('Computing GPG signature...');
+        console.log(`[${env.tag}] Computing GPG signature...`);
         safe.fs.unlinkSync(`${options.code}.sig`);
         execSync(`gpg --no-default-keyring --local-user 0EADB19CDDA23CD0FE71E3470A372F8703C493CC --output ${options.code}.sig --detach-sig ${options.code}`,
             { stdio: [ null, process.stdout, process.stderr ] } );
 
-        console.log('Uploading source code tarball and signature...');
+        console.log(`[${env.tag}] Uploading source code tarball and signature...`);
         const sourceTarballName = path.basename(options.code);
-        execSync(`rsync ${options.code} ubuntu@${env.releasesServer}:/home/ubuntu/releases/${sourceTarballName}`, { stdio: [ null, process.stdout, process.stderr ] } );
-        execSync(`rsync ${options.code}.sig ubuntu@${env.releasesServer}:/home/ubuntu/releases/${sourceTarballName}.sig`, { stdio: [ null, process.stdout, process.stderr ] } );
+        execSync(`rsync --progress ${options.code} ubuntu@${env.releasesServer}:/home/ubuntu/releases/${sourceTarballName}`, { stdio: [ null, process.stdout, process.stderr ] } );
+        execSync(`rsync --progress ${options.code}.sig ubuntu@${env.releasesServer}:/home/ubuntu/releases/${sourceTarballName}.sig`, { stdio: [ null, process.stdout, process.stderr ] } );
 
         options.code = `https://${env.releasesServer}/${sourceTarballName}`;
     }
@@ -240,7 +237,7 @@ async function createRelease(options) {
         releases[secondLastVersion].next = null;
         delete releases[lastVersion];
 
-        console.log('Reverting %s', lastVersion);
+        console.log(`[${env.tag}] Reverting ${lastVersion}`);
         return await verifyAndUpload(env, releases);
     }
 
@@ -372,7 +369,7 @@ async function listRelease(options) {
         t.newRow();
     }
 
-    console.log(`Selected environment: ${env.tag}\n`);
+    console.log(`Selected environment: [${env.tag}]\n`);
     console.log(t.toString());
 }
 
@@ -386,10 +383,10 @@ async function sync(options) {
     else if (destEnv.tag === 'dev') sourceEnv = ENVIRONMENTS['staging'];
     else throw new Error('Unable to determine source environment to sync from');
 
-    console.log(`Syncing ${sourceEnv.tag} to ${destEnv.tag}`);
+    console.log(`Syncing ${sourceEnv.tag} versions to ${destEnv.tag}`);
 
     const [getVersionsError, response] = await safe(superagent.get(sourceEnv.url));
-    if (getVersionsError) throw new Error(`Error getting versions.json: ${getVersionsError.message}`);
+    if (getVersionsError) throw new Error(`Error getting versions.json from ${sourceEnv.url}: ${getVersionsError}`);
 
     const sourceReleases = response.body;
     let destReleases = {};
@@ -564,13 +561,21 @@ async function e2e(options) {
     });
     if (!ok) return exit(new Error('doing nothing'));
 
-    await sync({ env: 'staging' });
-    await sync({ env: 'dev' });
-    await createRelease({ code: options.code, env: 'dev' });
-    await stage(ENVIRONMENTS['dev'], ENVIRONMENTS['staging'], null);
-    await rerelease({ env: 'staging' });
+    try {
+        await sync({ env: 'staging' });
+        await sync({ env: 'dev' });
+        await createRelease({ code: options.code, env: 'dev' });
+        await stage(ENVIRONMENTS['dev'], ENVIRONMENTS['staging'], null);
+        await rerelease({ env: 'staging' });
+    } catch (error) {
+        exit(error);
+    }
 }
 
+// happy eyeballs workaround. when there is no ipv6, nodejs timesout prematurely since the default for ipv4 is just 250ms
+// https://github.com/nodejs/node/issues/54359
+net.setDefaultAutoSelectFamilyAttemptTimeout(5000);
+
 program.command('amend')
     .option('--env <dev/staging/prod>', 'Environment (dev/staging/prod)', 'dev')
     .option('--code <tarball>', 'Source code url')

setup/start.sh

@@ -57,35 +57,15 @@ if ! grep -q userland-proxy /etc/systemd/system/docker.service.d/cloudron.conf;
     systemctl restart docker
 fi
 
-mkdir -p "${BOX_DATA_DIR}"
-mkdir -p "${APPS_DATA_DIR}"
-mkdir -p "${MAIL_DATA_DIR}"
+mkdir -p "${BOX_DATA_DIR}" "${APPS_DATA_DIR}" "${MAIL_DATA_DIR}"
 
 # keep these in sync with paths.js
 log "Ensuring directories"
 
-mkdir -p "${PLATFORM_DATA_DIR}/graphite"
-mkdir -p "${PLATFORM_DATA_DIR}/mysql"
-mkdir -p "${PLATFORM_DATA_DIR}/postgresql"
-mkdir -p "${PLATFORM_DATA_DIR}/mongodb"
-mkdir -p "${PLATFORM_DATA_DIR}/redis"
-mkdir -p "${PLATFORM_DATA_DIR}/tls"
-mkdir -p "${PLATFORM_DATA_DIR}/addons/mail/banner" \
-         "${PLATFORM_DATA_DIR}/addons/mail/dkim"
-mkdir -p "${PLATFORM_DATA_DIR}/collectd"
-mkdir -p "${PLATFORM_DATA_DIR}/logrotate.d"
-mkdir -p "${PLATFORM_DATA_DIR}/acme"
-mkdir -p "${PLATFORM_DATA_DIR}/backup"
-mkdir -p "${PLATFORM_DATA_DIR}/logs/backup" \
-         "${PLATFORM_DATA_DIR}/logs/updater" \
-         "${PLATFORM_DATA_DIR}/logs/tasks" \
-         "${PLATFORM_DATA_DIR}/logs/collectd"
-mkdir -p "${PLATFORM_DATA_DIR}/update"
+mkdir -p "${PLATFORM_DATA_DIR}/"{graphite,mysql,postgresql,mongodb,redis,tls,collectd,logrotate.d,acme,backup,update,firewall,sshfs,cifs,oidc,diskusage}
+mkdir -p "${PLATFORM_DATA_DIR}/addons/mail/"{banner,dkim}
+mkdir -p "${PLATFORM_DATA_DIR}/logs/"{backup,updater,tasks,collectd}
 mkdir -p "${PLATFORM_DATA_DIR}/sftp/ssh" # sftp keys
-mkdir -p "${PLATFORM_DATA_DIR}/firewall"
-mkdir -p "${PLATFORM_DATA_DIR}/sshfs"
-mkdir -p "${PLATFORM_DATA_DIR}/cifs"
-mkdir -p "${PLATFORM_DATA_DIR}/oidc"
 
 # ensure backups folder exists and is writeable
 mkdir -p /var/backups
@@ -231,11 +211,14 @@ if ! HOME=${HOME_DIR} BOX_ENV=cloudron DATABASE_URL=mysql://root:${mysql_root_pa
     exit 1
 fi
 
+# migrate disk usage cache file
+[[ -f "${PLATFORM_DATA_DIR}/diskusage.json" ]] && mv "${PLATFORM_DATA_DIR}/diskusage.json" "${PLATFORM_DATA_DIR}/diskusage/cache.json"
+
 log "Changing ownership"
 # note, change ownership after db migrate. this allow db migrate to move files around as root and then we can fix it up here
 # be careful of what is chown'ed here. subdirs like mysql,redis etc are owned by the containers and will stop working if perms change
 chown -R "${USER}" /etc/cloudron
-chown "${USER}:${USER}" -R "${PLATFORM_DATA_DIR}/nginx" "${PLATFORM_DATA_DIR}/collectd" "${PLATFORM_DATA_DIR}/addons" "${PLATFORM_DATA_DIR}/acme" "${PLATFORM_DATA_DIR}/backup" "${PLATFORM_DATA_DIR}/logs" "${PLATFORM_DATA_DIR}/update" "${PLATFORM_DATA_DIR}/sftp" "${PLATFORM_DATA_DIR}/firewall" "${PLATFORM_DATA_DIR}/sshfs" "${PLATFORM_DATA_DIR}/cifs" "${PLATFORM_DATA_DIR}/tls" "${PLATFORM_DATA_DIR}/oidc"
+chown "${USER}:${USER}" -R "${PLATFORM_DATA_DIR}/"{nginx,collectd,addons,acme,backup,logs,update,sftp,firewall,sshfs,cifs,tls,oidc,diskusage}
 chown "${USER}:${USER}" "${PLATFORM_DATA_DIR}/INFRA_VERSION" 2>/dev/null || true
 chown "${USER}:${USER}" "${PLATFORM_DATA_DIR}"
 chown "${USER}:${USER}" "${APPS_DATA_DIR}"

setup/start/systemd/box.service

@@ -15,7 +15,7 @@ ExecStart=/home/yellowtent/box/box.js
 ExecReload=/bin/kill -HUP $MAINPID
 ; we run commands like df which will parse properly only with correct locale
 ; add "oidc-provider:*" to DEBUG for OpenID debugging
-Environment="HOME=/home/yellowtent" "USER=yellowtent" "DEBUG=box:*,connect-lastmile,-box:ldap" "BOX_ENV=cloudron" "NODE_ENV=production" "LC_ALL=C" "AWS_SDK_JS_SUPPRESS_MAINTENANCE_MODE_MESSAGE=1"
+Environment="HOME=/home/yellowtent" "USER=yellowtent" "DEBUG=box:*,connect-lastmile,-box:ldap,-box:oidc" "BOX_ENV=cloudron" "NODE_ENV=production" "LC_ALL=C" "AWS_SDK_JS_SUPPRESS_MAINTENANCE_MODE_MESSAGE=1"
 ; kill apptask processes as well
 KillMode=control-group
 ; Do not kill this process on OOM. Children inherit this score. Do not set it to -1000 so that MemoryMax can keep working

src/applinks.js

@@ -6,8 +6,7 @@ exports = module.exports = {
     add,
     get,
     update,
-    remove,
-    getIcon
+    del,
 };
 
 const assert = require('assert'),
@@ -55,6 +54,25 @@ function validateUpstreamUri(upstreamUri) {
     return null;
 }
 
+function validateAccessRestriction(accessRestriction) {
+    assert.strictEqual(typeof accessRestriction, 'object');
+
+    if (accessRestriction === null) return null;
+
+    if (accessRestriction.users) {
+        if (!Array.isArray(accessRestriction.users)) return new BoxError(BoxError.BAD_FIELD, 'users array property required');
+        if (!accessRestriction.users.every(function (e) { return typeof e === 'string'; })) return new BoxError(BoxError.BAD_FIELD, 'All users have to be strings');
+    }
+
+    if (accessRestriction.groups) {
+        if (!Array.isArray(accessRestriction.groups)) return new BoxError(BoxError.BAD_FIELD, 'groups array property required');
+        if (!accessRestriction.groups.every(function (e) { return typeof e === 'string'; })) return new BoxError(BoxError.BAD_FIELD, 'All groups have to be strings');
+    }
+
+    // TODO: maybe validate if the users and groups actually exist
+    return null;
+}
+
 async function list() {
     const results = await database.query(`SELECT ${APPLINKS_FIELDS} FROM applinks ORDER BY upstreamUri`);
 
@@ -67,20 +85,18 @@ async function listByUser(user) {
     assert.strictEqual(typeof user, 'object');
 
     const result = await list();
-    return result.filter((app) => apps.canAccess(app, user));
+    return result.filter((link) => apps.canAccess(link, user));
 }
 
-async function detectMetaInfo(applink) {
-    assert.strictEqual(typeof applink, 'object');
+async function detectMetaInfo(upstreamUri) {
+    assert.strictEqual(typeof upstreamUri, 'string');
 
-    const [error, response] = await safe(superagent.get(applink.upstreamUri).set('User-Agent', 'Mozilla'));
+    const [error, response] = await safe(superagent.get(upstreamUri).set('User-Agent', 'Mozilla').timeout(10*1000));
     if (error || !response.text) {
         debug('detectMetaInfo: Unable to fetch upstreamUri to detect icon and title', error.statusCode);
-        return;
+        return null;
     }
 
-    if (applink.favicon && applink.label) return;
-
     // set redirected URI if any for favicon url
     const redirectUri = (response.redirects && response.redirects.length) ? response.redirects[0] : null;
 
@@ -89,73 +105,84 @@ async function detectMetaInfo(applink) {
       // No-op to skip console errors.
     });
 
-    const [jsdomError, dom] = await safe(jsdom.JSDOM.fromURL(applink.upstreamUri, { virtualConsole }));
-    if (jsdomError) console.error('detectMetaInfo: jsdomError', jsdomError);
-
-    if (!applink.icon && dom) {
-        let favicon = '';
-        if (dom.window.document.querySelector('link[rel="apple-touch-icon"]')) favicon = dom.window.document.querySelector('link[rel="apple-touch-icon"]').href;
-        if (!favicon && dom.window.document.querySelector('meta[name="msapplication-TileImage"]')) favicon = dom.window.document.querySelector('meta[name="msapplication-TileImage"]').content;
-        if (!favicon && dom.window.document.querySelector('link[rel="shortcut icon"]')) favicon = dom.window.document.querySelector('link[rel="shortcut icon"]').href;
-        if (!favicon && dom.window.document.querySelector('link[rel="icon"]')) {
-            let iconElements = dom.window.document.querySelectorAll('link[rel="icon"]');
-            if (iconElements.length) {
-                favicon = iconElements[0].href; // choose first one for a start
-
-                // check if we have sizes attributes and then choose the largest one
-                iconElements = Array.from(iconElements).filter(function (e) {
-                    return e.attributes.getNamedItem('sizes') && e.attributes.getNamedItem('sizes').value;
-                }).sort(function (a, b) {
-                    return parseInt(b.attributes.getNamedItem('sizes').value.split('x')[0]) - parseInt(a.attributes.getNamedItem('sizes').value.split('x')[0]);
-                });
-                if (iconElements.length) favicon = iconElements[0].href;
-            }
-        }
-        if (!favicon && dom.window.document.querySelector('meta[itemprop="image"]')) favicon = dom.window.document.querySelector('meta[itemprop="image"]').content;
-
-        if (favicon) {
-            favicon = new URL(favicon, redirectUri || applink.upstreamUri).toString();
-
-            debug(`detectMetaInfo: found icon: ${favicon}`);
-
-            const [error, response] = await safe(superagent.get(favicon));
-            if (error) debug(`Failed to fetch icon ${favicon}: `, error);
-            else if (response.ok && response.headers['content-type'].indexOf('image') !== -1) applink.icon = response.body || response.text;
-            else debug(`Failed to fetch icon ${favicon}: statusCode=${response.status}`);
-        }
-
-        if (!favicon) {
-            debug(`Unable to find a suitable icon for ${applink.upstreamUri}, try fallback favicon.ico`);
-
-            const [error, response] = await safe(superagent.get(applink.upstreamUri + '/favicon.ico'));
-            if (error) debug(`Failed to fetch icon ${favicon}: `, error);
-            else if (response.ok && response.headers['content-type'].indexOf('image') !== -1) applink.icon = response.body || response.text;
-            else debug(`Failed to fetch icon ${favicon}: statusCode=${response.status} content type ${response.headers['content-type']}`);
-        }
+    const [jsdomError, dom] = await safe(jsdom.JSDOM.fromURL(upstreamUri, { virtualConsole }));
+    if (jsdomError || !dom) {
+        console.error('detectMetaInfo: jsdomError', jsdomError);
+        return null;
     }
 
-    if (!applink.label) {
-        if (dom.window.document.querySelector('meta[property="og:title"]')) applink.label = dom.window.document.querySelector('meta[property="og:title"]').content;
-        else if (dom.window.document.querySelector('meta[property="og:site_name"]')) applink.label = dom.window.document.querySelector('meta[property="og:site_name"]').content;
-        else if (dom.window.document.title) applink.label = dom.window.document.title;
+    let icon = null, label = '';
+
+    // icon detection
+    let favicon = '';
+    if (dom.window.document.querySelector('link[rel="apple-touch-icon"]')) favicon = dom.window.document.querySelector('link[rel="apple-touch-icon"]').href;
+    if (!favicon && dom.window.document.querySelector('meta[name="msapplication-TileImage"]')) favicon = dom.window.document.querySelector('meta[name="msapplication-TileImage"]').content;
+    if (!favicon && dom.window.document.querySelector('link[rel="shortcut icon"]')) favicon = dom.window.document.querySelector('link[rel="shortcut icon"]').href;
+    if (!favicon && dom.window.document.querySelector('link[rel="icon"]')) {
+        let iconElements = dom.window.document.querySelectorAll('link[rel="icon"]');
+        if (iconElements.length) {
+            favicon = iconElements[0].href; // choose first one for a start
+
+            // check if we have sizes attributes and then choose the largest one
+            iconElements = Array.from(iconElements).filter(function (e) {
+                return e.attributes.getNamedItem('sizes') && e.attributes.getNamedItem('sizes').value;
+            }).sort(function (a, b) {
+                return parseInt(b.attributes.getNamedItem('sizes').value.split('x')[0]) - parseInt(a.attributes.getNamedItem('sizes').value.split('x')[0]);
+            });
+            if (iconElements.length) favicon = iconElements[0].href;
+        }
     }
+    if (!favicon && dom.window.document.querySelector('meta[itemprop="image"]')) favicon = dom.window.document.querySelector('meta[itemprop="image"]').content;
+
+    if (favicon) {
+        favicon = new URL(favicon, redirectUri || upstreamUri).toString();
+
+        debug(`detectMetaInfo: found icon: ${favicon}`);
+
+        const [error, response] = await safe(superagent.get(favicon).timeout(10*1000));
+        if (error) debug(`Failed to fetch icon ${favicon}: `, error);
+        else if (response.ok && response.headers['content-type'].indexOf('image') !== -1) icon = response.body || response.text;
+        else debug(`Failed to fetch icon ${favicon}: statusCode=${response.status}`);
+    }
+
+    if (!favicon) {
+        debug(`Unable to find a suitable icon for ${upstreamUri}, try fallback favicon.ico`);
+
+        const [error, response] = await safe(superagent.get(`${upstreamUri}/favicon.ico`).timeout(10*1000));
+        if (error) debug(`Failed to fetch icon ${favicon}: `, error);
+        else if (response.ok && response.headers['content-type'].indexOf('image') !== -1) icon = response.body || response.text;
+        else debug(`Failed to fetch icon ${favicon}: statusCode=${response.status} content type ${response.headers['content-type']}`);
+    }
+
+    // detect label
+    if (dom.window.document.querySelector('meta[property="og:title"]')) label = dom.window.document.querySelector('meta[property="og:title"]').content;
+    else if (dom.window.document.querySelector('meta[property="og:site_name"]')) label = dom.window.document.querySelector('meta[property="og:site_name"]').content;
+    else if (dom.window.document.title) label = dom.window.document.title;
+
+    return { icon, label };
 }
 
 async function add(applink) {
     assert.strictEqual(typeof applink, 'object');
-    assert.strictEqual(typeof applink.upstreamUri, 'string');
 
     debug(`add: ${applink.upstreamUri}`);
 
     let error = validateUpstreamUri(applink.upstreamUri);
     if (error) throw error;
 
+    error = validateAccessRestriction(applink.accessRestriction);
+    if (error) throw error;
+
     if (applink.icon) {
         if (!validator.isBase64(applink.icon)) throw new BoxError(BoxError.BAD_FIELD, 'icon is not base64');
         applink.icon = Buffer.from(applink.icon, 'base64');
     }
 
-    await detectMetaInfo(applink);
+    if (!applink.icon || !applink.label) {
+        const meta = await detectMetaInfo(applink.upstreamUri);
+        if (!applink.label) applink.label = meta?.label;
+        if (!applink.icon) applink.icon = meta?.icon;
+    }
 
     const data = {
         id: uuid.v4(),
@@ -186,49 +213,54 @@ async function get(applinkId) {
     return result[0];
 }
 
-async function update(applinkId, applink) {
-    assert.strictEqual(typeof applinkId, 'string');
+async function update(applink, data) {
     assert.strictEqual(typeof applink, 'object');
-    assert.strictEqual(typeof applink.upstreamUri, 'string');
+    assert.strictEqual(typeof data, 'object');
 
-    debug(`update: ${applink.upstreamUri}`);
-
-    let error = validateUpstreamUri(applink.upstreamUri);
-    if (error) throw error;
-
-    if (applink.icon) {
-        if (!validator.isBase64(applink.icon)) throw new BoxError(BoxError.BAD_FIELD, 'icon is not base64');
-        applink.icon = Buffer.from(applink.icon, 'base64');
-    } else if (applink.icon === '') {
-        // empty string means we autodetect in detectMetaInfo
-        applink.icon = '';
-    } else {
-        // nothing changed reuse old
-        const result = await get(applinkId);
-        applink.icon = result.icon;
+    let error;
+    if ('upstreamUri' in data) {
+        error = validateUpstreamUri(data.upstreamUri);
+        if (error) throw error;
     }
 
-    await detectMetaInfo(applink);
+    if ('accessRestriction' in data) {
+        error = validateAccessRestriction(data.accessRestriction);
+        if (error) throw error;
+    }
 
-    const query = 'UPDATE applinks SET label=?, icon=?, upstreamUri=?, tagsJson=?, accessRestrictionJson=? WHERE id = ?';
-    const args = [ applink.label, applink.icon || null, applink.upstreamUri, applink.tags ? JSON.stringify(applink.tags) : null, applink.accessRestriction ? JSON.stringify(applink.accessRestriction) : null, applinkId ];
+    if ('icon' in data && data.icon) {
+        if (!validator.isBase64(data.icon)) throw new BoxError(BoxError.BAD_FIELD, 'icon is not base64');
+        data.icon = Buffer.from(data.icon, 'base64');
+    }
 
-    const result = await database.query(query, args);
+    // we don't track if label/icon in db is user-set or was auto detected
+    if (data.upstreamUri || data.label === '' || data.icon === '') {
+        const meta = await detectMetaInfo(data.upstreamUri || applink.upstreamUri);
+
+        if (data.label === '') data.label = meta?.label;
+        if (data.icon === '') data.icon = meta?.icon;
+    }
+
+    const args = [], fields = [];
+    for (const k in data) {
+        if (k === 'accessRestriction' || k === 'tags') {
+            fields.push(`${k}Json = ?`);
+            args.push(JSON.stringify(data[k]));
+        } else {
+            fields.push(k + ' = ?');
+            args.push(data[k]);
+        }
+    }
+    args.push(applink.id);
+
+    const [updateError, result] = await safe(database.query('UPDATE applinks SET ' + fields.join(', ') + ' WHERE id = ?', args));
+    if (updateError) throw updateError;
     if (result.affectedRows !== 1) throw new BoxError(BoxError.NOT_FOUND, 'Applink not found');
 }
 
-async function remove(applinkId) {
-    assert.strictEqual(typeof applinkId, 'string');
+async function del(applink) {
+    assert.strictEqual(typeof applink, 'object');
 
-    const result = await database.query('DELETE FROM applinks WHERE id = ?', [ applinkId ]);
+    const result = await database.query('DELETE FROM applinks WHERE id = ?', [ applink.id ]);
     if (result.affectedRows !== 1) throw new BoxError(BoxError.NOT_FOUND, 'Applink not found');
 }
-
-async function getIcon(applinkId) {
-    assert.strictEqual(typeof applinkId, 'string');
-
-    const applink = await get(applinkId);
-    if (!applink) throw new BoxError(BoxError.NOT_FOUND, 'Applink not found');
-
-    return applink.icon;
-}

src/apps.js

@@ -21,7 +21,9 @@ exports = module.exports = {
 
     // user actions
     install,
+    unarchive,
     uninstall,
+    archive,
 
     setAccessRestriction,
     setOperators,
@@ -35,6 +37,7 @@ exports = module.exports = {
     setMemoryLimit,
     setCpuQuota,
     setMounts,
+    setDevices,
     setAutomaticBackup,
     setAutomaticUpdate,
     setReverseProxyConfig,
@@ -144,6 +147,7 @@ exports = module.exports = {
 
 const appstore = require('./appstore.js'),
     appTaskManager = require('./apptaskmanager.js'),
+    archives = require('./archives.js'),
     assert = require('assert'),
     backups = require('./backups.js'),
     BoxError = require('./boxerror.js'),
@@ -182,10 +186,11 @@ const appstore = require('./appstore.js'),
     volumes = require('./volumes.js'),
     _ = require('underscore');
 
+// NOTE: when adding fields here, update the clone and unarchive logic as well
 const APPS_FIELDS_PREFIXED = [ 'apps.id', 'apps.appStoreId', 'apps.installationState', 'apps.errorJson', 'apps.runState',
     'apps.health', 'apps.containerId', 'apps.manifestJson', 'apps.accessRestrictionJson', 'apps.memoryLimit', 'apps.cpuQuota',
     'apps.label', 'apps.notes', 'apps.tagsJson', 'apps.taskId', 'apps.reverseProxyConfigJson', 'apps.servicesConfigJson', 'apps.operatorsJson',
-    'apps.sso', 'apps.debugModeJson', 'apps.enableBackup', 'apps.proxyAuth', 'apps.containerIp', 'apps.crontab',
+    'apps.sso', 'apps.devicesJson', 'apps.debugModeJson', 'apps.enableBackup', 'apps.proxyAuth', 'apps.containerIp', 'apps.crontab',
     'apps.creationTime', 'apps.updateTime', 'apps.enableAutomaticUpdate', 'apps.upstreamUri', 'apps.checklistJson',
     'apps.enableMailbox', 'apps.mailboxDisplayName', 'apps.mailboxName', 'apps.mailboxDomain', 'apps.enableInbox', 'apps.inboxName', 'apps.inboxDomain',
     'apps.enableTurn', 'apps.enableRedis', 'apps.storageVolumeId', 'apps.storageVolumePrefix', 'apps.ts', 'apps.healthTime', '(apps.icon IS NOT NULL) AS hasIcon', '(apps.appStoreIcon IS NOT NULL) AS hasAppStoreIcon' ].join(',');
@@ -483,6 +488,14 @@ function validateTags(tags) {
     return null;
 }
 
+function validateDevices(devices) {
+    for (const key in devices) {
+        if (key.indexOf('/dev/') !== 0) return new BoxError(BoxError.BAD_FIELD, `"${key}" must start with /dev/`);
+    }
+
+    return null;
+}
+
 function validateEnv(env) {
     for (const key in env) {
         if (key.length > 512) return new BoxError(BoxError.BAD_FIELD, 'Max env var key length is 512');
@@ -581,7 +594,7 @@ function removeInternalFields(app) {
         'subdomain', 'domain', 'fqdn', 'certificate', 'crontab', 'upstreamUri',
         'accessRestriction', 'manifest', 'portBindings', 'iconUrl', 'memoryLimit', 'cpuQuota', 'operators',
         'sso', 'debugMode', 'reverseProxyConfig', 'enableBackup', 'creationTime', 'updateTime', 'ts', 'tags',
-        'label', 'notes', 'secondaryDomains', 'redirectDomains', 'aliasDomains', 'env', 'enableAutomaticUpdate',
+        'label', 'notes', 'secondaryDomains', 'redirectDomains', 'aliasDomains', 'devices', 'env', 'enableAutomaticUpdate',
         'storageVolumeId', 'storageVolumePrefix', 'mounts', 'enableTurn', 'enableRedis', 'checklist',
         'enableMailbox', 'mailboxDisplayName', 'mailboxName', 'mailboxDomain', 'enableInbox', 'inboxName', 'inboxDomain');
 
@@ -740,6 +753,16 @@ function postProcess(result) {
     delete result.errorJson;
 
     result.taskId = result.taskId ? String(result.taskId) : null;
+
+    // result.devices = {
+    //     '/dev/ttyUSB10': {
+    //         // future options
+    //     },
+    //     '/dev/hidraw0': {}
+    // };
+
+    result.devices = result.devicesJson ? JSON.parse(result.devicesJson) : {};
+    delete result.devicesJson;
 }
 
 // attaches computed properties
@@ -836,14 +859,15 @@ async function add(id, appStoreId, manifest, subdomain, domain, portBindings, da
     assert(data && typeof data === 'object');
 
     const manifestJson = JSON.stringify(manifest),
-        accessRestriction = data.accessRestriction || null,
-        accessRestrictionJson = JSON.stringify(accessRestriction),
+        accessRestrictionJson = data.accessRestriction ? JSON.stringify(data.accessRestriction) : null,
+        operatorsJson = data.operators ? JSON.stringify(data.operators) : null,
         memoryLimit = data.memoryLimit || 0,
         cpuQuota = data.cpuQuota || 100,
         installationState = data.installationState,
         runState = data.runState,
         sso = 'sso' in data ? data.sso : null,
         debugModeJson = data.debugMode ? JSON.stringify(data.debugMode) : null,
+        devicesJson = data.devices ? JSON.stringify(data.devices) : null,
         env = data.env || {},
         label = data.label || null,
         tagsJson = data.tags ? JSON.stringify(data.tags) : null,
@@ -857,20 +881,26 @@ async function add(id, appStoreId, manifest, subdomain, domain, portBindings, da
         upstreamUri = data.upstreamUri || '',
         enableTurn = 'enableTurn' in data ? data.enableTurn : true,
         enableRedis = 'enableRedis' in data ? data.enableRedis : true,
-        icon = data.icon || null;
+        icon = data.icon || null,
+        notes = data.notes || null,
+        crontab = data.crontab || null,
+        enableBackup = 'enableBackup' in data ? data.enableBackup : true,
+        enableAutomaticUpdate = 'enableAutomaticUpdate' in data ? data.enableAutomaticUpdate : true;
 
     await checkForPortBindingConflict(portBindings, { appId: null });
 
     const queries = [];
 
     queries.push({
-        query: 'INSERT INTO apps (id, appStoreId, manifestJson, installationState, runState, accessRestrictionJson, memoryLimit, cpuQuota, '
+        query: 'INSERT INTO apps (id, appStoreId, manifestJson, installationState, runState, accessRestrictionJson, operatorsJson, memoryLimit, cpuQuota, '
             + 'sso, debugModeJson, mailboxName, mailboxDomain, label, tagsJson, reverseProxyConfigJson, checklistJson, servicesConfigJson, icon, '
-            + 'enableMailbox, mailboxDisplayName, upstreamUri, enableTurn, enableRedis) '
-            + ' VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
-        args: [ id, appStoreId, manifestJson, installationState, runState, accessRestrictionJson, memoryLimit, cpuQuota,
+            + 'enableMailbox, mailboxDisplayName, upstreamUri, enableTurn, enableRedis, devicesJson, notes, crontab, enableBackup, enableAutomaticUpdate) '
+            + ' VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
+        args: [ id, appStoreId, manifestJson, installationState, runState, accessRestrictionJson, operatorsJson, memoryLimit, cpuQuota,
             sso, debugModeJson, mailboxName, mailboxDomain, label, tagsJson, reverseProxyConfigJson, checklistJson, servicesConfigJson, icon,
-            enableMailbox, mailboxDisplayName, upstreamUri, enableTurn, enableRedis ]
+            enableMailbox, mailboxDisplayName, upstreamUri, enableTurn, enableRedis, devicesJson, notes, crontab,
+            enableBackup, enableAutomaticUpdate
+        ]
     });
 
     queries.push({
@@ -1004,7 +1034,7 @@ async function updateWithConstraints(id, app, constraints) {
 
     const fields = [ ], values = [ ];
     for (const p in app) {
-        if (p === 'manifest' || p === 'tags' || p === 'checklist' || p === 'accessRestriction' || p === 'debugMode' || p === 'error' || p === 'reverseProxyConfig' || p === 'servicesConfig' || p === 'operators') {
+        if (p === 'manifest' || p === 'tags' || p === 'checklist' || p === 'accessRestriction' || p === 'devices' || p === 'debugMode' || p === 'error' || p === 'reverseProxyConfig' || p === 'servicesConfig' || p === 'operators') {
             fields.push(`${p}Json = ?`);
             values.push(JSON.stringify(app[p]));
         } else if (p !== 'portBindings' && p !== 'subdomain' && p !== 'domain' && p !== 'secondaryDomains' && p !== 'redirectDomains' && p !== 'aliasDomains' && p !== 'env' && p !== 'mounts') {
@@ -1178,10 +1208,11 @@ async function onTaskFinished(error, appId, installationState, taskId, auditSour
         const toManifest = success ? app.manifest : task.args[1].updateConfig.manifest;
 
         await eventlog.add(eventlog.ACTION_APP_UPDATE_FINISH, auditSource, { app, toManifest, fromManifest, success, errorMessage });
+        await notifications.unpin(notifications.TYPE_MANUAL_APP_UPDATE_NEEDED, { context: app.id });
         break;
     }
     case exports.ISTATE_PENDING_BACKUP: {
-        const backup = await backups.get(task.result);
+        const backup = task.result ? await backups.get(task.result) : null; // if task crashed, no result
         await eventlog.add(eventlog.ACTION_APP_BACKUP_FINISH, auditSource, { app, success, errorMessage, remotePath: backup?.remotePath, backupId: task.result });
         break;
     }
@@ -1315,12 +1346,15 @@ async function install(data, auditSource) {
     const subdomain = data.subdomain.toLowerCase(),
         domain = data.domain.toLowerCase(),
         accessRestriction = data.accessRestriction || null,
+        operators = data.operators || null,
         memoryLimit = data.memoryLimit || 0,
+        cpuQuota = data.cpuQuota || 100,
         debugMode = data.debugMode || null,
         enableBackup = 'enableBackup' in data ? data.enableBackup : true,
         enableAutomaticUpdate = 'enableAutomaticUpdate' in data ? data.enableAutomaticUpdate : true,
         redirectDomains = data.redirectDomains || [],
         aliasDomains = data.aliasDomains || [],
+        devices = data.devices || {},
         env = data.env || {},
         label = data.label || null,
         tags = data.tags || [],
@@ -1330,7 +1364,9 @@ async function install(data, auditSource) {
         enableRedis = 'enableRedis' in data ? data.enableRedis : true,
         appStoreId = data.appStoreId,
         upstreamUri = data.upstreamUri || '',
-        manifest = data.manifest;
+        manifest = data.manifest,
+        notes = data.notes || null,
+        crontab = data.crontab || null;
 
     let error = manifestFormat.parse(manifest);
     if (error) throw new BoxError(BoxError.BAD_FIELD, `Manifest error: ${error.message}`);
@@ -1345,6 +1381,9 @@ async function install(data, auditSource) {
     error = validateAccessRestriction(accessRestriction);
     if (error) throw error;
 
+    error = validateAccessRestriction(operators); // not a typo. same structure for operators and accessRestriction
+    if (error) throw error;
+
     error = validateMemoryLimit(manifest, memoryLimit);
     if (error) throw error;
 
@@ -1354,6 +1393,11 @@ async function install(data, auditSource) {
     error = validateLabel(label);
     if (error) throw error;
 
+    error = validateCpuQuota(cpuQuota);
+    if (error) throw error;
+
+    parseCrontab(crontab);
+
     if ('upstreamUri' in data) error = validateUpstreamUri(upstreamUri);
     if (error) throw error;
 
@@ -1369,6 +1413,9 @@ async function install(data, auditSource) {
     // if sso was unspecified, enable it by default if possible
     if (sso === null) sso = !!manifest.addons?.ldap || !!manifest.addons?.proxyAuth || !!manifest.addons?.oidc;
 
+    error = validateDevices(devices);
+    if (error) throw error;
+
     error = validateEnv(env);
     if (error) throw error;
 
@@ -1396,11 +1443,13 @@ async function install(data, auditSource) {
     if (constants.DEMO && (await getCount() >= constants.DEMO_APP_LIMIT)) throw new BoxError(BoxError.BAD_STATE, 'Too many installed apps, please uninstall a few and try again');
 
     const appId = uuid.v4();
-    debug('Will install app with id : ' + appId);
+    debug(`Installing app ${appId}`);
 
     const app = {
         accessRestriction,
+        operators,
         memoryLimit,
+        cpuQuota,
         sso,
         debugMode,
         mailboxName,
@@ -1411,6 +1460,7 @@ async function install(data, auditSource) {
         redirectDomains,
         aliasDomains,
         env,
+        devices,
         label,
         tags,
         icon,
@@ -1418,6 +1468,8 @@ async function install(data, auditSource) {
         upstreamUri,
         enableTurn,
         enableRedis,
+        notes,
+        crontab,
         runState: exports.RSTATE_RUNNING,
         installationState: exports.ISTATE_PENDING_INSTALL
     };
@@ -1642,6 +1694,30 @@ async function setMounts(app, mounts, auditSource) {
     return { taskId };
 }
 
+async function setDevices(app, devices, auditSource) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof devices, 'object');
+    assert.strictEqual(typeof auditSource, 'object');
+
+    const appId = app.id;
+    let error = checkAppState(app, exports.ISTATE_PENDING_RECREATE_CONTAINER);
+    if (error) throw error;
+
+    error = validateDevices(devices);
+    if (error) throw error;
+
+    const task = {
+        args: {},
+        values: { devices }
+    };
+    const [taskError, taskId] = await safe(addTask(appId, exports.ISTATE_PENDING_RECREATE_CONTAINER, task, auditSource));
+    if (taskError) throw taskError;
+
+    await eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId, app, devices, taskId });
+
+    return { taskId };
+}
+
 async function setEnvironment(app, env, auditSource) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof env, 'object');
@@ -2362,11 +2438,11 @@ async function clone(app, data, user, auditSource) {
 
     const newAppId = uuid.v4();
 
-    const icons = await getIcons(app.id);
-
-    const dolly = _.pick(app, 'memoryLimit', 'cpuQuota', 'crontab', 'reverseProxyConfig', 'env', 'servicesConfig', 'tags',
-        'enableMailbox', 'mailboxDisplayName', 'mailboxName', 'mailboxDomain', 'enableInbox', 'inboxName', 'inboxDomain',
-        'enableTurn', 'enableRedis', 'mounts', 'enableBackup', 'enableAutomaticUpdate', 'accessRestriction', 'operators', 'sso');
+    // label, checklist intentionally omitted . icon is loaded in apptask from the backup
+    const dolly = _.pick(backupInfo.appConfig || app, 'memoryLimit', 'cpuQuota', 'crontab', 'reverseProxyConfig', 'env', 'servicesConfig', 'tags', 'devices',
+        'enableMailbox', 'mailboxDisplayName', 'mailboxName', 'mailboxDomain', 'enableInbox', 'inboxName', 'inboxDomain', 'debugMode',
+        'enableTurn', 'enableRedis', 'mounts', 'enableBackup', 'enableAutomaticUpdate', 'accessRestriction', 'operators', 'sso',
+        'notes', 'checklist');
 
     if (!manifest.addons?.recvmail) dolly.inboxDomain = null; // needed because we are cloning _current_ app settings with old manifest
 
@@ -2378,8 +2454,7 @@ async function clone(app, data, user, auditSource) {
         secondaryDomains,
         redirectDomains: [],
         aliasDomains: [],
-        label: app.label ? `${app.label}-clone` : '',
-        icon: icons.icon,
+        label: dolly.label ? `${dolly.label}-clone` : '',
     });
 
     const [addError] = await safe(add(newAppId, appStoreId, manifest, subdomain, domain, portBindings, obj));
@@ -2407,6 +2482,82 @@ async function clone(app, data, user, auditSource) {
     return { id: newAppId, taskId };
 }
 
+async function unarchive(archive, data, auditSource) {
+    assert.strictEqual(typeof archive, 'object');
+    assert.strictEqual(typeof data, 'object');
+    assert(auditSource && typeof auditSource === 'object');
+
+    const backup = await backups.get(archive.backupId);
+    const restoreConfig = { remotePath: backup.remotePath, backupFormat: backup.format };
+
+    const subdomain = data.subdomain.toLowerCase(),
+        domain = data.domain.toLowerCase(),
+        overwriteDns = 'overwriteDns' in data ? data.overwriteDns : false;
+
+    const appConfig = backup.appConfig;
+    const { appStoreId, manifest } = appConfig;
+
+    let error = validateSecondaryDomains(data.secondaryDomains || {}, manifest);
+    if (error) throw error;
+    const secondaryDomains = translateSecondaryDomains(data.secondaryDomains || {});
+
+    const locations = [new Location(subdomain, domain, Location.TYPE_PRIMARY)]
+        .concat(secondaryDomains.map(sd => new Location(sd.subdomain, sd.domain, Location.TYPE_SECONDARY)));
+
+    error = await validateLocations(locations);
+    if (error) throw error;
+
+    // re-validate because this new box version may not accept old configs
+    error = await checkManifest(manifest);
+    if (error) throw error;
+
+    error = validatePorts(data.ports || null, manifest);
+    if (error) throw error;
+    const portBindings = translateToPortBindings(data.ports || null, manifest);
+
+    const appId = uuid.v4();
+
+    const dolly = _.pick(appConfig, 'memoryLimit', 'cpuQuota', 'crontab', 'reverseProxyConfig', 'env', 'servicesConfig',
+        'tags', 'label', 'enableMailbox', 'mailboxDisplayName', 'mailboxName', 'mailboxDomain', 'enableInbox', 'inboxName', 'inboxDomain', 'devices',
+        'enableTurn', 'enableRedis', 'mounts', 'enableBackup', 'enableAutomaticUpdate', 'accessRestriction', 'operators', 'sso',
+        'notes', 'checklist');
+
+    // intentionally not filled up: redirectDomain, aliasDomains, mailboxDomain
+    const obj = Object.assign(dolly, {
+        secondaryDomains,
+        redirectDomains: [],
+        aliasDomains: [],
+        mailboxDomain: data.domain, // archive's mailboxDomain may not exist
+        runState: exports.RSTATE_RUNNING,
+        installationState: exports.ISTATE_PENDING_INSTALL
+    });
+    obj.icon = (await archives.getIcons(archive.id))?.icon;
+
+    const [addError] = await safe(add(appId, appStoreId, manifest, subdomain, domain, portBindings, obj));
+    if (addError && addError.reason === BoxError.ALREADY_EXISTS) throw getDuplicateErrorDetails(addError.message, locations, portBindings);
+    if (addError) throw addError;
+
+    await purchaseApp({ appId, appstoreId: appStoreId, manifestId: manifest.id || 'customapp' });
+
+    const task = {
+        args: { restoreConfig, overwriteDns },
+        values: {},
+        requiredState: obj.installationState
+    };
+
+    const taskId = await addTask(appId, obj.installationState, task, auditSource);
+
+    const newApp = Object.assign({}, _.omit(obj, 'icon'), { appStoreId, manifest, subdomain, domain, portBindings });
+    newApp.fqdn = dns.fqdn(newApp.subdomain, newApp.domain);
+    newApp.secondaryDomains.forEach(function (ad) { ad.fqdn = dns.fqdn(ad.subdomain, ad.domain); });
+    newApp.redirectDomains.forEach(function (ad) { ad.fqdn = dns.fqdn(ad.subdomain, ad.domain); });
+    newApp.aliasDomains.forEach(function (ad) { ad.fqdn = dns.fqdn(ad.subdomain, ad.domain); });
+
+    await eventlog.add(eventlog.ACTION_APP_INSTALL, auditSource, { appId, app: newApp, taskId });
+
+    return { id : appId, taskId };
+}
+
 async function uninstall(app, auditSource) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof auditSource, 'object');
@@ -2428,6 +2579,23 @@ async function uninstall(app, auditSource) {
     return { taskId };
 }
 
+async function archive(app, backupId, auditSource) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof backupId, 'string');
+    assert.strictEqual(typeof auditSource, 'object');
+
+    if (app.manifest.id === constants.PROXY_APP_APPSTORE_ID) throw new BoxError(BoxError.BAD_FIELD, 'cannot archive proxy app');
+
+    const result = await backups.getByIdentifierAndStatePaged(app.id, backups.BACKUP_STATE_NORMAL, 1, 1);
+    if (result.length === 0) throw new BoxError(BoxError.BAD_STATE, 'No recent backup to archive');
+    if (result[0].id !== backupId) throw new BoxError(BoxError.BAD_STATE, 'Latest backup id has changed');
+
+    const icons = await getIcons(app.id);
+    const archiveId = await archives.add(backupId, { icon: icons.icon, appStoreIcon: icons.appStoreIcon, appConfig: app }, auditSource);
+    const { taskId } = await uninstall(app, auditSource);
+    return { taskId, id: archiveId };
+}
+
 async function start(app, auditSource) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof auditSource, 'object');
@@ -2619,7 +2787,8 @@ async function autoupdateApps(updateInfo, auditSource) { // updateInfo is { appI
 
         if (!canAutoupdateApp(app, updateInfo[appId])) {
             debug(`app ${app.fqdn} requires manual update`);
-            notifications.alert(notifications.ALERT_MANUAL_APP_UPDATE, `${app.manifest.title} at ${app.fqdn} requires manual update to version ${updateInfo[appId].manifest.version}`, `Changelog:\n${updateInfo[appId].manifest.changelog}\n`, { persist: false });
+            notifications.pin(notifications.TYPE_MANUAL_APP_UPDATE_NEEDED, `${app.manifest.title} at ${app.fqdn} requires manual update to version ${updateInfo[appId].manifest.version}`,
+                `Changelog:\n${updateInfo[appId].manifest.changelog}\n`, { context: app.id });
             continue;
         }
 

src/apptask.js

@@ -315,7 +315,7 @@ async function install(app, args, progressCallback) {
     }
 
     if (oldManifest && oldManifest.dockerImage !== app.manifest.dockerImage) {
-        await docker.deleteImage(oldManifest);
+        await docker.deleteImage(oldManifest.dockerImage);
     }
 
     // allocating container ip here, lets the users "repair" an app if allocation fails at apps.add time
@@ -366,6 +366,10 @@ async function install(app, args, progressCallback) {
         await services.setupAddons(app, app.manifest.addons);
         await services.clearAddons(app, app.manifest.addons);
         await backuptask.downloadApp(app, restoreConfig, (progress) => { progressCallback({ percent: 65, message: progress.message }); });
+        if (app.installationState === apps.ISTATE_PENDING_CLONE) {
+            const customIcon = safe.fs.readFileSync(path.join(paths.APPS_DATA_DIR, app.id + '/icon.png'));
+            if (customIcon) await updateApp(app, { icon: customIcon });
+        }
         await progressCallback({ percent: 70, message: 'Restoring addons' });
         await services.restoreAddons(app, app.manifest.addons);
     }
@@ -628,7 +632,7 @@ async function update(app, args, progressCallback) {
     // we cannot easily 'recover' from backup failures because we have to revert manfest and portBindings
     await progressCallback({ percent: 35, message: 'Deleting old containers' });
     await deleteContainers(app, { managedOnly: true });
-    if (app.manifest.dockerImage !== updateConfig.manifest.dockerImage) await docker.deleteImage(app.manifest);
+    if (app.manifest.dockerImage !== updateConfig.manifest.dockerImage) await docker.deleteImage(app.manifest.dockerImage);
 
     // only delete unused addons after backup
     await services.teardownAddons(app, unusedAddons);
@@ -767,7 +771,7 @@ async function uninstall(app, args, progressCallback) {
     await deleteAppDir(app, { removeDirectory: true });
 
     await progressCallback({ percent: 60, message: 'Deleting image' });
-    await docker.deleteImage(app.manifest);
+    await docker.deleteImage(app.manifest.dockerImage);
 
     await progressCallback({ percent: 70, message: 'Unregistering domains' });
     await dns.unregisterLocations([ { subdomain: app.subdomain, domain: app.domain } ].concat(app.secondaryDomains).concat(app.redirectDomains).concat(app.aliasDomains), progressCallback);

src/apptaskmanager.js

@@ -1,6 +1,7 @@
 'use strict';
 
 exports = module.exports = {
+    start,
     scheduleTask
 };
 
@@ -8,84 +9,83 @@ const assert = require('assert'),
     BoxError = require('./boxerror.js'),
     debug = require('debug')('box:apptaskmanager'),
     fs = require('fs'),
-    locker = require('./locker.js'),
-    safe = require('safetydance'),
+    locks = require('./locks.js'),
     path = require('path'),
     paths = require('./paths.js'),
+    safe = require('safetydance'),
     scheduler = require('./scheduler.js'),
     tasks = require('./tasks.js');
 
 const gActiveTasks = {}; // indexed by app id
 const gPendingTasks = [];
-let gInitialized = false;
+let gStarted = false;
 
 const TASK_CONCURRENCY = 3;
+const DRAIN_TIMER_SECS = 1000;
 
-function waitText(lockOperation) {
-    if (lockOperation === locker.OP_BOX_UPDATE) return 'Waiting for Cloudron to finish updating. See the Settings view';
-    if (lockOperation === locker.OP_INFRA_START) return 'Waiting for Platform Services to start. See the Services view';
-    if (lockOperation === locker.OP_FULL_BACKUP) return 'Waiting for Cloudron to finish backup. See the Backups view';
+let gDrainTimerId = null;
 
-    return ''; // cannot happen
+async function drain() {
+    debug(`drain: ${gPendingTasks.length} apptasks pending`);
+
+    for (let i = 0; i < gPendingTasks.length; i++) {
+        const space = Object.keys(gActiveTasks).length - TASK_CONCURRENCY;
+        if (space == 0) {
+            debug('At concurrency limit, cannot drain anymore');
+            break;
+        }
+
+        const { appId, taskId, options, onFinished }  = gPendingTasks[i];
+
+        const [lockError] = await safe(locks.acquire(`${locks.TYPE_APP_PREFIX}${appId}`));
+        if (lockError) continue;
+
+        gPendingTasks.splice(i, 1);
+        gActiveTasks[appId] = {};
+
+        const logFile = path.join(paths.LOG_DIR, appId, 'apptask.log');
+
+        if (!fs.existsSync(path.dirname(logFile))) safe.fs.mkdirSync(path.dirname(logFile)); // ensure directory
+
+        scheduler.suspendJobs(appId);
+
+        tasks.startTask(taskId, Object.assign(options, { logFile }), async function (error, result) {
+            onFinished(error, result);
+
+            delete gActiveTasks[appId];
+            await locks.release(`${locks.TYPE_APP_PREFIX}${appId}`);
+            scheduler.resumeJobs(appId);
+        });
+    }
+
+    gDrainTimerId = null;
+    if (gPendingTasks.length) gDrainTimerId = setTimeout(drain, DRAIN_TIMER_SECS); // check for released locks
 }
 
-function initializeSync() {
-    gInitialized = true;
-    locker.on('unlocked', startNextTask);
+async function start() {
+    assert.strictEqual(gDrainTimerId, null);
+    assert.strictEqual(gStarted, false);
+
+    debug('started');
+    gStarted = true;
+
+    if (gPendingTasks.length) gDrainTimerId = setTimeout(drain, DRAIN_TIMER_SECS);
 }
 
-// callback is called when task is finished
 function scheduleTask(appId, taskId, options, onFinished) {
     assert.strictEqual(typeof appId, 'string');
     assert.strictEqual(typeof taskId, 'string');
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof onFinished, 'function');
 
-    if (!gInitialized) initializeSync();
-
     if (appId in gActiveTasks) {
-        return onFinished(new BoxError(BoxError.CONFLICT, `Task for ${appId} is already active`));
-    }
-
-    if (Object.keys(gActiveTasks).length >= TASK_CONCURRENCY) {
-        debug(`Reached concurrency limit, queueing task id ${taskId}`);
-        tasks.update(taskId, { percent: 1, message: 'Waiting for other app tasks to complete' });
-        gPendingTasks.push({ appId, taskId, options, onFinished });
+        onFinished(new BoxError(BoxError.CONFLICT, `Task for ${appId} is already active`));
         return;
     }
 
-    const lockError = locker.recursiveLock(locker.OP_APPTASK);
+    // percent 1 is relies on the tasks "active" flag to indicate task is queued but not started yet
+    tasks.update(taskId, { percent: 1, message: gStarted ? 'Queued' : 'Waiting for platform to initialize' });
+    gPendingTasks.push({ appId, taskId, options, onFinished });
 
-    if (lockError) {
-        debug(`Could not get lock. ${lockError.message}, queueing task id ${taskId}`);
-        tasks.update(taskId, { percent: 1, message: waitText(lockError.operation) });
-        gPendingTasks.push({ appId, taskId, options, onFinished });
-        return;
-    }
-
-    gActiveTasks[appId] = {};
-
-    const logFile = path.join(paths.LOG_DIR, appId, 'apptask.log');
-
-    if (!fs.existsSync(path.dirname(logFile))) safe.fs.mkdirSync(path.dirname(logFile)); // ensure directory
-
-    scheduler.suspendJobs(appId);
-
-    tasks.startTask(taskId, Object.assign(options, { logFile }), function (error, result) {
-        onFinished(error, result);
-
-        delete gActiveTasks[appId];
-        locker.unlock(locker.OP_APPTASK); // unlock event will trigger next task
-
-        scheduler.resumeJobs(appId);
-    });
-}
-
-function startNextTask() {
-    if (gPendingTasks.length === 0) return;
-
-    assert(Object.keys(gActiveTasks).length < TASK_CONCURRENCY);
-
-    const t = gPendingTasks.shift();
-    scheduleTask(t.appId, t.taskId, t.options, t.onFinished);
+    if (gStarted && !gDrainTimerId) gDrainTimerId = setTimeout(drain, DRAIN_TIMER_SECS);
 }

src/archives.js

@@ -0,0 +1,106 @@
+'use strict';
+
+exports = module.exports = {
+    get,
+    getIcons,
+    getIcon,
+    add,
+    list,
+    listBackupIds,
+    del,
+};
+
+const assert = require('assert'),
+    BoxError = require('./boxerror.js'),
+    database = require('./database.js'),
+    eventlog = require('./eventlog.js'),
+    safe = require('safetydance'),
+    uuid = require('uuid');
+
+const ARCHIVE_FIELDS = [ 'archives.id', 'backupId', 'archives.creationTime', 'backups.remotePath', 'backups.appConfigJson', '(archives.icon IS NOT NULL) AS hasIcon', '(archives.appStoreIcon IS NOT NULL) AS hasAppStoreIcon' ];
+
+function postProcess(result) {
+    assert.strictEqual(typeof result, 'object');
+
+    result.appConfig = result.appConfigJson ? safe.JSON.parse(result.appConfigJson) : null;
+    delete result.appConfigJson;
+
+    result.iconUrl = result.hasIcon || result.hasAppStoreIcon  ? `/api/v1/archives/${result.id}/icon` : null;
+
+    return result;
+}
+
+async function get(id) {
+    assert.strictEqual(typeof id, 'string');
+
+    const result = await database.query(`SELECT ${ARCHIVE_FIELDS} FROM archives LEFT JOIN backups ON archives.backupId = backups.id  WHERE archives.id = ? ORDER BY creationTime DESC`, [ id ]);
+    if (result.length === 0) return null;
+
+    return postProcess(result[0]);
+}
+
+async function getIcons(id) {
+    assert.strictEqual(typeof id, 'string');
+
+    const results = await database.query('SELECT icon, appStoreIcon FROM archives WHERE id=?', [ id ]);
+    if (results.length === 0) return null;
+    return { icon: results[0].icon, appStoreIcon: results[0].appStoreIcon };
+}
+
+async function getIcon(id, options) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof options, 'object');
+
+    const icons = await getIcons(id);
+    if (!icons) throw new BoxError(BoxError.NOT_FOUND, 'No such backup');
+
+    if (!options.original && icons.icon) return icons.icon;
+    if (icons.appStoreIcon) return icons.appStoreIcon;
+
+    return null;
+}
+
+async function add(backupId, data, auditSource) {
+    assert.strictEqual(typeof backupId, 'string');
+    assert.strictEqual(typeof data, 'object');
+    assert(auditSource && typeof auditSource === 'object');
+
+    const id = uuid.v4();
+
+    const [error] = await safe(database.query('INSERT INTO archives (id, backupId, icon, appStoreIcon) VALUES (?, ?, ?, ?)',
+        [ id, backupId, data.icon, data.appStoreIcon ]));
+
+    if (error && error.code === 'ER_NO_REFERENCED_ROW_2') throw new BoxError(BoxError.NOT_FOUND, 'Backup not found');
+    if (error && error.code === 'ER_DUP_ENTRY') throw new BoxError(BoxError.ALREADY_EXISTS, 'Archive already exists');
+    if (error) throw error;
+
+    await eventlog.add(eventlog.ACTION_ARCHIVES_ADD, auditSource, { id, backupId });
+
+    return id;
+}
+
+async function list(page, perPage) {
+    assert(typeof page === 'number' && page > 0);
+    assert(typeof perPage === 'number' && perPage > 0);
+
+    const results = await database.query(`SELECT ${ARCHIVE_FIELDS} FROM archives LEFT JOIN backups ON archives.backupId = backups.id ORDER BY creationTime DESC LIMIT ?,?`, [ (page-1)*perPage, perPage ]);
+
+    results.forEach(function (result) { postProcess(result); });
+
+    return results;
+}
+
+async function listBackupIds() {
+    const results = await database.query(`SELECT backupId FROM archives`, []);
+    return results.map(r => r.backupId);
+}
+
+async function del(archive, auditSource) {
+    assert.strictEqual(typeof archive, 'object');
+    assert(auditSource && typeof auditSource === 'object');
+
+    const result = await database.query('DELETE FROM archives WHERE id=?', [ archive.id ]);
+    if (result.affectedRows !== 1) throw new BoxError(BoxError.NOT_FOUND, 'No such archive');
+
+    await eventlog.add(eventlog.ACTION_ARCHIVES_DEL, auditSource, { id: archive.id, backupId: archive.backupId });
+}

src/backupcleaner.js

@@ -9,6 +9,7 @@ exports = module.exports = {
 };
 
 const apps = require('./apps.js'),
+    archives = require('./archives.js'),
     assert = require('assert'),
     backupFormat = require('./backupformat.js'),
     backups = require('./backups.js'),
@@ -33,7 +34,7 @@ function applyBackupRetention(allBackups, retention, referencedBackupIds) {
         } else if (backup.state === backups.BACKUP_STATE_CREATING) {
             if ((now - backup.creationTime) < 48*60*60*1000) backup.keepReason = 'creating';
             else backup.discardReason = 'creating-too-long';
-        } else if (referencedBackupIds.includes(backup.id)) {
+        } else if (referencedBackupIds.includes(backup.id)) { // could also be in archives
             backup.keepReason = 'referenced';
         } else if ((backup.preserveSecs === -1) || ((now - backup.creationTime) < (backup.preserveSecs * 1000))) {
             backup.keepReason = 'preserveSecs';
@@ -74,7 +75,7 @@ function applyBackupRetention(allBackups, retention, referencedBackupIds) {
     }
 
     for (const backup of allBackups) {
-        debug(`applyBackupRetentionPolicy: ${backup.remotePath} keep/discard: ${backup.keepReason || backup.discardReason || 'unprocessed'}`);
+        debug(`applyBackupRetention: ${backup.remotePath} keep/discard: ${backup.keepReason || backup.discardReason || 'unprocessed'}`);
     }
 }
 
@@ -295,7 +296,8 @@ async function run(progressCallback) {
     const removedMailBackupPaths = await cleanupMailBackups(backupConfig, retention, referencedBackupIds, progressCallback);
 
     await progressCallback({ percent: 40, message: 'Cleaning app backups' });
-    const removedAppBackupPaths = await cleanupAppBackups(backupConfig, retention, referencedBackupIds, progressCallback);
+    const archivedBackupIds = await archives.listBackupIds();
+    const removedAppBackupPaths = await cleanupAppBackups(backupConfig, retention, referencedBackupIds.concat(archivedBackupIds), progressCallback);
 
     await progressCallback({ percent: 70, message: 'Checking storage backend and removing stale entries in database' });
     const missingBackupPaths = await cleanupMissingBackups(backupConfig, progressCallback);

src/backupformat/rsync.js

@@ -123,15 +123,24 @@ async function saveFsMetadata(dataLayout, metadataFile) {
         symlinks: []
     };
 
+    const MAX_FILES = 20000; // this is just a rough upper bound
+
     // we assume small number of files. spawnSync will raise a ENOBUFS error after maxBuffer
     for (const lp of dataLayout.localPaths()) {
-        const emptyDirs = await shell.spawn('find', [lp, '-type', 'd', '-empty'], { encoding: 'utf8', maxBuffer: 1024 * 1024 * 80 });
+        const [emptyDirsError, emptyDirs] = await safe(shell.spawn('find', [lp, '-type', 'd', '-empty'], { encoding: 'utf8', maxLines: MAX_FILES }));
+        if (emptyDirsError && emptyDirsError.stdoutLineCount >= MAX_FILES) throw new BoxError(BoxError.FS_ERROR, `Too many empty directories. Run "find ${lp} -type d -empty" to investigate`);
+        if (emptyDirsError) throw emptyDirsError;
         if (emptyDirs.length) metadata.emptyDirs = metadata.emptyDirs.concat(emptyDirs.trim().split('\n').map((ed) => dataLayout.toRemotePath(ed)));
 
-        const execFiles = await shell.spawn('find', [lp, '-type', 'f', '-executable'], { encoding: 'utf8', maxBuffer: 1024 * 1024 * 80 });
+        const [execFilesError, execFiles] = await safe(shell.spawn('find', [lp, '-type', 'f', '-executable'], { encoding: 'utf8', maxLines: MAX_FILES }));
+        if (execFilesError && execFilesError.stdoutLineCount >= MAX_FILES) throw new BoxError(BoxError.FS_ERROR, `Too many executable files. Run "find ${lp} -type f -executable" to investigate`);
+        if (execFilesError) throw execFilesError;
         if (execFiles.length) metadata.execFiles = metadata.execFiles.concat(execFiles.trim().split('\n').map((ef) => dataLayout.toRemotePath(ef)));
 
-        const symlinkFiles = await shell.spawn('find', [lp, '-type', 'l'], { encoding: 'utf8', maxBuffer: 1024 * 1024 * 30 });
+        const [symlinkFilesError, symlinkFiles] = await safe(shell.spawn('find', [lp, '-type', 'l'], { encoding: 'utf8', maxLines: MAX_FILES }));
+        if (symlinkFilesError && symlinkFilesError.stdoutLineCount >= MAX_FILES) throw new BoxError(BoxError.FS_ERROR, `Too many symlinks. Run "find ${lp} -type l" to investigate`);
+        if (symlinkFilesError) throw symlinkFilesError;
+
         if (symlinkFiles.length) metadata.symlinks = metadata.symlinks.concat(symlinkFiles.trim().split('\n').map((sl) => {
             const target = safe.fs.readlinkSync(sl);
             return { path: dataLayout.toRemotePath(sl), target };

src/backups.js

@@ -63,7 +63,7 @@ const assert = require('assert'),
     debug = require('debug')('box:backups'),
     eventlog = require('./eventlog.js'),
     hat = require('./hat.js'),
-    locker = require('./locker.js'),
+    locks = require('./locks.js'),
     mounts = require('./mounts.js'),
     path = require('path'),
     paths = require('./paths.js'),
@@ -73,7 +73,7 @@ const assert = require('assert'),
     tasks = require('./tasks.js'),
     _ = require('underscore');
 
-const BACKUPS_FIELDS = [ 'id', 'remotePath', 'label', 'identifier', 'creationTime', 'packageVersion', 'type', 'dependsOnJson', 'state', 'manifestJson', 'format', 'preserveSecs', 'encryptionVersion' ];
+const BACKUPS_FIELDS = [ 'id', 'remotePath', 'label', 'identifier', 'creationTime', 'packageVersion', 'type', 'dependsOnJson', 'state', 'manifestJson', 'format', 'preserveSecs', 'encryptionVersion', 'appConfigJson' ];
 
 function postProcess(result) {
     assert.strictEqual(typeof result, 'object');
@@ -84,6 +84,9 @@ function postProcess(result) {
     result.manifest = result.manifestJson ? safe.JSON.parse(result.manifestJson) : null;
     delete result.manifestJson;
 
+    result.appConfig = result.appConfigJson ? safe.JSON.parse(result.appConfigJson) : null;
+    delete result.appConfigJson;
+
     return result;
 }
 
@@ -103,10 +106,10 @@ function generateEncryptionKeysSync(password) {
 
     const aesKeys = crypto.scryptSync(password, Buffer.from('CLOUDRONSCRYPTSALT', 'utf8'), 128);
     return {
-        dataKey: aesKeys.slice(0, 32).toString('hex'),
-        dataHmacKey: aesKeys.slice(32, 64).toString('hex'),
-        filenameKey: aesKeys.slice(64, 96).toString('hex'),
-        filenameHmacKey: aesKeys.slice(96).toString('hex')
+        dataKey: aesKeys.subarray(0, 32).toString('hex'),
+        dataHmacKey: aesKeys.subarray(32, 64).toString('hex'),
+        filenameKey: aesKeys.subarray(64, 96).toString('hex'),
+        filenameHmacKey: aesKeys.subarray(96).toString('hex')
     };
 }
 
@@ -122,14 +125,16 @@ async function add(data) {
     assert.strictEqual(typeof data.manifest, 'object');
     assert.strictEqual(typeof data.format, 'string');
     assert.strictEqual(typeof data.preserveSecs, 'number');
+    assert.strictEqual(typeof data.appConfig, 'object');
 
     const creationTime = data.creationTime || new Date(); // allow tests to set the time
     const manifestJson = JSON.stringify(data.manifest);
     const prefixId = data.type === exports.BACKUP_TYPE_APP ? `${data.type}_${data.identifier}` : data.type; // type and identifier are same for other types
     const id = `${prefixId}_v${data.packageVersion}_${hat(32)}`; // id is used by the UI to derive dependent packages. making this a UUID will require a lot of db querying
+    const appConfigJson = data.appConfig ? JSON.stringify(data.appConfig) : null;
 
-    const [error] = await safe(database.query('INSERT INTO backups (id, remotePath, identifier, encryptionVersion, packageVersion, type, creationTime, state, dependsOnJson, manifestJson, format, preserveSecs) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
-        [ id, data.remotePath, data.identifier, data.encryptionVersion, data.packageVersion, data.type, creationTime, data.state, JSON.stringify(data.dependsOn), manifestJson, data.format, data.preserveSecs ]));
+    const [error] = await safe(database.query('INSERT INTO backups (id, remotePath, identifier, encryptionVersion, packageVersion, type, creationTime, state, dependsOnJson, manifestJson, format, preserveSecs, appConfigJson) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
+        [ id, data.remotePath, data.identifier, data.encryptionVersion, data.packageVersion, data.type, creationTime, data.state, JSON.stringify(data.dependsOn), manifestJson, data.format, data.preserveSecs, appConfigJson ]));
 
     if (error && error.code === 'ER_DUP_ENTRY') throw new BoxError(BoxError.ALREADY_EXISTS, 'Backup already exists');
     if (error) throw error;
@@ -239,8 +244,8 @@ async function setState(id, state) {
 }
 
 async function startBackupTask(auditSource) {
-    const error = locker.lock(locker.OP_FULL_BACKUP);
-    if (error) throw new BoxError(BoxError.BAD_STATE, `Cannot backup now: ${error.message}`);
+    const [error] = await safe(locks.acquire(locks.TYPE_BACKUP_TASK));
+    if (error) throw new BoxError(BoxError.BAD_STATE, `Another backup task is in progress: ${error.message}`);
 
     const backupConfig = await getConfig();
 
@@ -251,7 +256,8 @@ async function startBackupTask(auditSource) {
     await eventlog.add(eventlog.ACTION_BACKUP_START, auditSource, { taskId });
 
     tasks.startTask(taskId, { timeout: 24 * 60 * 60 * 1000 /* 24 hours */, nice: 15, memoryLimit, oomScoreAdjust: -999 }, async function (error, backupId) {
-        locker.unlock(locker.OP_FULL_BACKUP);
+        await locks.release(locks.TYPE_BACKUP_TASK);
+        await locks.releaseByTaskId(taskId);
 
         const errorMessage = error ? error.message : '';
         const timedOut = error ? error.code === tasks.ETIMEOUT : false;

src/backuptask.js

@@ -24,6 +24,7 @@ const apps = require('./apps.js'),
     database = require('./database.js'),
     debug = require('debug')('box:backuptask'),
     df = require('./df.js'),
+    locks = require('./locks.js'),
     path = require('path'),
     paths = require('./paths.js'),
     safe = require('safetydance'),
@@ -117,8 +118,9 @@ async function restore(backupConfig, remotePath, progressCallback) {
     debug('restore: download completed, importing database');
 
     await database.importFromFile(`${dataLayout.localRoot()}/box.mysqldump`);
-
     debug('restore: database imported');
+
+    await locks.releaseAll();
 }
 
 async function downloadApp(app, restoreConfig, progressCallback) {
@@ -250,7 +252,8 @@ async function rotateBoxBackup(backupConfig, tag, options, dependsOn, progressCa
         dependsOn,
         manifest: null,
         format,
-        preserveSecs: options.preserveSecs || 0
+        preserveSecs: options.preserveSecs || 0,
+        appConfig: null
     };
 
     const id = await backups.add(data);
@@ -299,7 +302,8 @@ async function rotateAppBackup(backupConfig, app, tag, options, progressCallback
         dependsOn: [],
         manifest,
         format,
-        preserveSecs: options.preserveSecs || 0
+        preserveSecs: options.preserveSecs || 0,
+        appConfig: app
     };
 
     const id = await backups.add(data);
@@ -434,7 +438,8 @@ async function rotateMailBackup(backupConfig, tag, options, progressCallback) {
         dependsOn: [],
         manifest: null,
         format,
-        preserveSecs: options.preserveSecs || 0
+        preserveSecs: options.preserveSecs || 0,
+        appConfig: null
     };
 
     const id = await backups.add(data);
@@ -496,7 +501,6 @@ async function fullBackup(options, progressCallback) {
     const appBackupIds = [];
     for (let i = 0; i < allApps.length; i++) {
         const app = allApps[i];
-        progressCallback({ percent: percent, message: `Backing up ${app.fqdn} (${i+1}/${allApps.length})` });
         percent += step;
 
         if (!app.enableBackup) {
@@ -504,9 +508,13 @@ async function fullBackup(options, progressCallback) {
             continue; // nothing to backup
         }
 
+        progressCallback({ percent, message: `Backing up ${app.fqdn} (${i+1}/${allApps.length}). Waiting for lock` });
+        await locks.wait(`${locks.TYPE_APP_PREFIX}${app.id}`);
         const startTime = new Date();
-        const appBackupId = await backupAppWithTag(app, tag, options, (progress) => progressCallback({ percent, message: progress.message }));
+        const [appBackupError, appBackupId] = await safe(backupAppWithTag(app, tag, options, (progress) => progressCallback({ percent, message: progress.message })));
         debug(`fullBackup: app ${app.fqdn} backup finished. Took ${(new Date() - startTime)/1000} seconds`);
+        await locks.release(`${locks.TYPE_APP_PREFIX}${app.id}`);
+        if (appBackupError) throw appBackupError;
         if (appBackupId) appBackupIds.push(appBackupId); // backupId can be null if in BAD_STATE and never backed up
     }
 

src/boxerror.js

@@ -10,7 +10,7 @@ exports = module.exports = BoxError;
 
 function BoxError(reason, errorOrMessage, extra = {}) {
     assert.strictEqual(typeof reason, 'string');
-    assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string');
+    assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string', `string: ${errorOrMessage} type: ${typeof errorOrMessage} json: ${JSON.stringify(errorOrMessage)}`);
     assert(typeof extra === 'object');
 
     Error.call(this);

src/branding.js

@@ -16,9 +16,12 @@ exports = module.exports = {
     renderFooter
 };
 
-const assert = require('assert'),
+const apps = require('./apps.js'),
+    assert = require('assert'),
     BoxError = require('./boxerror.js'),
     constants = require('./constants.js'),
+    debug = require('debug')('box:branding'),
+    eventlog = require('./eventlog.js'),
     paths = require('./paths.js'),
     safe = require('safetydance'),
     settings = require('./settings.js');
@@ -28,8 +31,9 @@ async function getCloudronName() {
     return name || 'Cloudron';
 }
 
-async function setCloudronName(name) {
+async function setCloudronName(name, auditSource) {
     assert.strictEqual(typeof name, 'string');
+    assert(auditSource && typeof auditSource === 'object');
 
     if (!name) throw new BoxError(BoxError.BAD_FIELD, 'name is empty');
 
@@ -37,7 +41,12 @@ async function setCloudronName(name) {
     // if this is changed, adjust dashboard/branding.html
     if (name.length > 64) throw new BoxError(BoxError.BAD_FIELD, 'name cannot exceed 64 characters');
 
+    // mark apps using oidc addon to be reconfigured
+    const [, installedApps] = await safe(apps.list());
+    await safe(apps.configureApps(installedApps.filter((a) => !!a.manifest.addons?.oidc), { scheduleNow: true }, auditSource), { debug });
+
     await settings.set(settings.CLOUDRON_NAME_KEY, name);
+    await eventlog.add(eventlog.ACTION_BRANDING_NAME, auditSource, { name });
 }
 
 async function getCloudronAvatar() {
@@ -51,14 +60,16 @@ async function getCloudronAvatar() {
     throw new BoxError(BoxError.FS_ERROR, `Could not read avatar: ${safe.error.message}`);
 }
 
-async function setCloudronAvatar(avatar) {
+async function setCloudronAvatar(avatar, auditSource) {
     assert(Buffer.isBuffer(avatar));
+    assert(auditSource && typeof auditSource === 'object');
 
     await settings.setBlob(settings.CLOUDRON_AVATAR_KEY, avatar);
+    await eventlog.add(eventlog.ACTION_BRANDING_AVATAR, auditSource, {});
 }
 
 async function getCloudronBackground() {
-    let background = await settings.getBlob(settings.CLOUDRON_BACKGROUND_KEY);
+    const background = await settings.getBlob(settings.CLOUDRON_BACKGROUND_KEY);
     if (!background) return null;
 
     return background;
@@ -83,8 +94,10 @@ async function getFooter() {
     return value || constants.FOOTER;
 }
 
-async function setFooter(footer) {
+async function setFooter(footer, auditSource) {
     assert.strictEqual(typeof footer, 'string');
+    assert(auditSource && typeof auditSource === 'object');
 
     await settings.set(settings.FOOTER_KEY, footer);
+    await eventlog.add(eventlog.ACTION_BRANDING_FOOTER, auditSource, { footer });
 }

src/df.js

@@ -1,7 +1,7 @@
 'use strict';
 
 exports = module.exports = {
-    disks,
+    filesystems,
     file,
     prettyBytes
 };
@@ -36,10 +36,10 @@ function parseLine(line) {
     };
 }
 
-async function disks() {
+async function filesystems() {
     const [error, output] = await safe(shell.spawn('df', ['-B1', '--output=source,fstype,size,used,avail,pcent,target'], { encoding: 'utf8', timeout: 5000 }));
     if (error) {
-        debug(`disks: df command failed. error: ${error}\n stdout: ${error.stdout}\n stderr: ${error.stderr}`);
+        debug(`filesystems: df command failed. error: ${error}\n stdout: ${error.stdout}\n stderr: ${error.stderr}`);
         throw new BoxError(BoxError.FS_ERROR, `Error running df: ${error.message}`);
     }
 

src/dns.js

@@ -53,6 +53,7 @@ function api(provider) {
     case 'digitalocean': return require('./dns/digitalocean.js');
     case 'gandi': return require('./dns/gandi.js');
     case 'godaddy': return require('./dns/godaddy.js');
+    case 'inwx': return require('./dns/inwx.js');
     case 'linode': return require('./dns/linode.js');
     case 'vultr': return require('./dns/vultr.js');
     case 'namecom': return require('./dns/namecom.js');
@@ -153,7 +154,7 @@ async function upsertDnsRecords(subdomain, domain, type, values) {
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
 
-    debug(`upsertDNSRecord: location ${subdomain} on domain ${domain} of type ${type} with values ${JSON.stringify(values)}`);
+    debug(`upsertDnsRecords: subdomain:${subdomain} domain:${domain} type:${type} values:${JSON.stringify(values)}`);
 
     const domainObject = await domains.get(domain);
     await api(domainObject.provider).upsert(domainObject, subdomain, type, values);
@@ -165,7 +166,7 @@ async function removeDnsRecords(subdomain, domain, type, values) {
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
 
-    debug('removeDNSRecords: %s on %s type %s values', subdomain, domain, type, values);
+    debug(`removeDnsRecords: subdomain:${subdomain} domain:${domain} type:${type} values:${JSON.stringify(values)}`);
 
     const domainObject = await domains.get(domain);
     const [error] = await safe(api(domainObject.provider).del(domainObject, subdomain, type, values));
@@ -324,7 +325,13 @@ async function syncDnsRecords(options, progressCallback) {
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof progressCallback, 'function');
 
-    if (options.domain && options.type === 'mail') return await mail.setDnsRecords(options.domain);
+    const errors = [];
+
+    if (options.domain && options.type === 'mail') {
+        const [error] = await safe(mail.setDnsRecords(options.domain));
+        if (error) errors.push({ domain: options.domain, error: error.message });
+        return errors;
+    }
 
     let allDomains = await domains.list();
 
@@ -333,7 +340,7 @@ async function syncDnsRecords(options, progressCallback) {
     const { domain:mailDomain, fqdn:mailFqdn, subdomain:mailSubdomain } = await mailServer.getLocation();
     const dashboardLocation = await dashboard.getLocation();
 
-    const allApps = await apps.list(), errors = [];
+    const allApps = await apps.list();
     let progress = 1;
 
     // we sync by domain only to get some nice progress

src/dns/bunny.js

@@ -53,7 +53,7 @@ async function getZoneId(domainConfig, zoneName) {
         .retry(5)
         .timeout(30 * 1000)
         .ok(() => true));
-    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
     if (response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
     if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
 
@@ -78,7 +78,7 @@ async function getDnsRecords(domainConfig, zoneName, name, type) {
         .retry(5)
         .timeout(30 * 1000)
         .ok(() => true));
-    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
     if (response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
     if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
     if (!Array.isArray(response.body.Records)) throw new BoxError(BoxError.EXTERNAL_ERROR, `Invalid records in response: ${JSON.stringify(response.body)}`);
@@ -129,7 +129,7 @@ async function upsert(domainObject, location, type, values) {
                 .retry(5)
                 .timeout(30 * 1000)
                 .ok(() => true));
-            if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+            if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
             if (response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
             if (response.statusCode !== 201) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
         } else {
@@ -141,7 +141,7 @@ async function upsert(domainObject, location, type, values) {
                 .ok(() => true));
             ++i;
 
-            if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+            if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
             if (response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
             if (response.statusCode !== 204) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
 
@@ -198,7 +198,7 @@ async function del(domainObject, location, type, values) {
             .timeout(30 * 1000)
             .ok(() => true));
 
-        if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+        if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
         if (response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
         if (response.statusCode === 400) continue;
         if (response.statusCode !== 204) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));

src/dns/cloudflare.js

@@ -75,7 +75,7 @@ async function getZoneByName(domainConfig, zoneName) {
     assert.strictEqual(typeof zoneName, 'string');
 
     const [error, response] = await safe(createRequest('GET', `${CLOUDFLARE_ENDPOINT}/zones?name=${zoneName}&status=active`, domainConfig));
-    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
     if (response.statusCode !== 200 || response.body.success !== true) throw translateRequestError(response);
     if (!response.body.result || !response.body.result.length) throw new BoxError(BoxError.NOT_FOUND, `${response.statusCode} ${response.text}`);
 
@@ -99,7 +99,7 @@ async function getDnsRecords(domainConfig, zoneId, fqdn, type) {
     const [error, response] = await safe(createRequest('GET', `${CLOUDFLARE_ENDPOINT}/zones/${zoneId}/dns_records`, domainConfig)
         .query({ type: type, name: fqdn }));
 
-    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
     if (response.statusCode !== 200 || response.body.success !== true) throw translateRequestError(response);
 
     const result = response.body.result;
@@ -155,7 +155,7 @@ async function upsert(domainObject, location, type, values) {
 
             const [error, response] = await safe(createRequest('POST', `${CLOUDFLARE_ENDPOINT}/zones/${zoneId}/dns_records`, domainConfig)
                 .send(data));
-            if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+            if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
             if (response.statusCode !== 200 || response.body.success !== true) throw translateRequestError(response);
         } else { // replace existing record
             data.proxied = records[i].proxied; // preserve proxied parameter
@@ -164,13 +164,14 @@ async function upsert(domainObject, location, type, values) {
 
             const [error, response] = await safe(createRequest('PUT', `${CLOUDFLARE_ENDPOINT}/zones/${zoneId}/dns_records/${records[i].id}`, domainConfig)
                 .send(data));
-            if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+            if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
             if (response.statusCode !== 200 || response.body.success !== true) throw translateRequestError(response);
             ++i; // increment, as we have consumed the record
         }
     }
 
     for (let j = values.length + 1; j < records.length; j++) {
+
         const [error] = await safe(createRequest('DELETE', `${CLOUDFLARE_ENDPOINT}/zones/${zoneId}/dns_records/${records[j].id}`, domainConfig));
         if (error) debug(`upsert: error removing record ${records[j].id}: ${error.message}`);
     }
@@ -215,7 +216,7 @@ async function del(domainObject, location, type, values) {
 
     for (const r of tmp) {
         const [error, response] = await safe(createRequest('DELETE', `${CLOUDFLARE_ENDPOINT}/zones/${zoneId}/dns_records/${r.id}`, domainConfig));
-        if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+        if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
         if (response.statusCode !== 200 || response.body.success !== true) throw translateRequestError(response);
     }
 }

src/dns/desec.js

@@ -54,7 +54,7 @@ async function get(domainObject, location, type) {
         .retry(5)
         .ok(() => true));
 
-    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
     if (response.statusCode === 404) return [];
     if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
     if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
@@ -90,7 +90,7 @@ async function upsert(domainObject, location, type, values) {
         .retry(5)
         .ok(() => true));
 
-    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
     if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
     if (response.statusCode !== 201) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
 }
@@ -112,7 +112,7 @@ async function del(domainObject, location, type, values) {
         .retry(5)
         .ok(() => true));
 
-    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
     if (response.statusCode === 404) return;
     if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
     if (response.statusCode !== 204) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));

src/dns/digitalocean.js

@@ -54,7 +54,7 @@ async function getZoneRecords(domainConfig, zoneName, name, type) {
             .retry(5)
             .ok(() => true));
 
-        if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+        if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
         if (response.statusCode === 404) throw new BoxError(BoxError.NOT_FOUND, formatError(response));
         if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
         if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
@@ -84,7 +84,8 @@ async function upsert(domainObject, location, type, values) {
     const records = await getZoneRecords(domainConfig, zoneName, name, type);
 
     // used to track available records to update instead of create
-    let i = 0, recordIds = [];
+    let i = 0;
+    const recordIds = [];
 
     for (let value of values) {
         let priority = null;
@@ -112,7 +113,7 @@ async function upsert(domainObject, location, type, values) {
                 .retry(5)
                 .ok(() => true));
 
-            if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+            if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
             if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
             if (response.statusCode === 422) throw new BoxError(BoxError.BAD_FIELD, response.body.message);
             if (response.statusCode !== 201) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
@@ -128,7 +129,7 @@ async function upsert(domainObject, location, type, values) {
 
             ++i;
 
-            if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+            if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
             if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
             if (response.statusCode === 422) throw new BoxError(BoxError.BAD_FIELD, response.body.message);
             if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
@@ -188,7 +189,7 @@ async function del(domainObject, location, type, values) {
             .retry(5)
             .ok(() => true));
 
-        if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+        if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
         if (response.statusCode === 404) return;
         if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
         if (response.statusCode !== 204) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));

src/dns/dnsimple.js

@@ -43,7 +43,7 @@ async function getAccountId(domainConfig) {
         .retry(5)
         .timeout(30 * 1000)
         .ok(() => true));
-    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
     if (response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
     if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
 
@@ -62,7 +62,7 @@ async function getZone(domainConfig, zoneName) {
         .retry(5)
         .timeout(30 * 1000)
         .ok(() => true));
-    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
     if (response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
     if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
 
@@ -87,7 +87,7 @@ async function getDnsRecords(domainConfig, zoneName, name, type) {
         .retry(5)
         .timeout(30 * 1000)
         .ok(() => true));
-    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
     if (response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
     if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
     if (!Array.isArray(response.body.data)) throw new BoxError(BoxError.EXTERNAL_ERROR, `Invalid data in response: ${JSON.stringify(response.body)}`);
@@ -138,7 +138,7 @@ async function upsert(domainObject, location, type, values) {
                 .retry(5)
                 .timeout(30 * 1000)
                 .ok(() => true));
-            if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+            if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
             if (response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
             if (response.statusCode !== 201) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
             recordIds.push(safe.query(response.body, 'data.id'));
@@ -151,7 +151,7 @@ async function upsert(domainObject, location, type, values) {
                 .ok(() => true));
             ++i;
 
-            if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+            if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
             if (response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
             if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
             recordIds.push(safe.query(response.body, 'data.id'));
@@ -207,7 +207,7 @@ async function del(domainObject, location, type, values) {
             .timeout(30 * 1000)
             .ok(() => true));
 
-        if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+        if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
         if (response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
         if (response.statusCode === 404) continue;
         if (response.statusCode !== 204) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));

src/dns/gandi.js

@@ -74,7 +74,7 @@ async function upsert(domainObject, location, type, values) {
     const [error, response] = await safe(createRequest('PUT', `${GANDI_API}/domains/${zoneName}/records/${name}/${type}`, domainConfig)
         .send(data));
 
-    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
     if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
     if (response.statusCode === 400) throw new BoxError(BoxError.BAD_FIELD, formatError(response));
     if (response.statusCode !== 201) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
@@ -93,7 +93,7 @@ async function get(domainObject, location, type) {
 
     const [error, response] = await safe(createRequest('GET', `${GANDI_API}/domains/${zoneName}/records/${name}/${type}`, domainConfig));
 
-    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
     if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
     if (response.statusCode === 404) return [];
     if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
@@ -115,7 +115,7 @@ async function del(domainObject, location, type, values) {
 
     const [error, response] = await safe(createRequest('DELETE', `${GANDI_API}/domains/${zoneName}/records/${name}/${type}`, domainConfig));
 
-    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
     if (response.statusCode === 404) return;
     if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
     if (response.statusCode !== 204) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));

src/dns/godaddy.js

@@ -69,7 +69,7 @@ async function upsert(domainObject, location, type, values) {
         .send(records)
         .ok(() => true));
 
-    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
     if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
     if (response.statusCode === 400) throw new BoxError(BoxError.BAD_FIELD, formatError(response)); // no such zone
     if (response.statusCode === 422) throw new BoxError(BoxError.BAD_FIELD, formatError(response)); // conflict
@@ -92,7 +92,7 @@ async function get(domainObject, location, type) {
         .timeout(30 * 1000)
         .ok(() => true));
 
-    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
     if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
     if (response.statusCode === 404) return [];
     if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
@@ -138,7 +138,7 @@ async function del(domainObject, location, type, values) {
         .timeout(30 * 1000)
         .ok(() => true));
 
-    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
     if (response.statusCode === 404) return;
     if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
     if (response.statusCode !== 204) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));

src/dns/hetzner.js

@@ -45,7 +45,7 @@ async function getZone(domainConfig, zoneName) {
         .timeout(30 * 1000)
         .retry(5)
         .ok(() => true));
-    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
     if (response.statusCode === 401 || response.statusCode === 403) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
     if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
 
@@ -64,11 +64,10 @@ async function getZoneRecords(domainConfig, zone, name, type) {
 
     let page = 1, matchingRecords = [];
 
-    debug(`getInternal: getting dns records of ${zone.name} with ${name} and type ${type}`);
+    debug(`getZoneRecords: getting dns records of ${zone.name} with ${name} and type ${type}`);
 
     const perPage = 50;
 
-    // eslint-disable-next-line no-constant-condition
     while (true) {
         const [error, response] = await safe(superagent.get(`${ENDPOINT}/records`)
             .set('Auth-API-Token', domainConfig.token)
@@ -77,7 +76,7 @@ async function getZoneRecords(domainConfig, zone, name, type) {
             .retry(5)
             .ok(() => true));
 
-        if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+        if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
         if (response.statusCode === 404) throw new BoxError(BoxError.NOT_FOUND, formatError(response));
         if (response.statusCode === 401 || response.statusCode === 403) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
         if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
@@ -104,7 +103,7 @@ async function upsert(domainObject, location, type, values) {
         zoneName = domainObject.zoneName,
         name = dns.getName(domainObject, location, type) || '@';
 
-    debug('upsert: %s for zone %s of type %s with values %j', name, zoneName, type, values);
+    debug(`upsert: ${name} for zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
 
     const zone = await getZone(domainConfig, zoneName);
     const records = await getZoneRecords(domainConfig, zone, name, type);
@@ -112,7 +111,7 @@ async function upsert(domainObject, location, type, values) {
     // used to track available records to update instead of create
     let i = 0;
 
-    for (let value of values) {
+    for (const value of values) {
         const data = {
             type,
             name,
@@ -129,7 +128,7 @@ async function upsert(domainObject, location, type, values) {
                 .retry(5)
                 .ok(() => true));
 
-            if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+            if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
             if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
             if (response.statusCode === 422) throw new BoxError(BoxError.BAD_FIELD, response.body.message);
             if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
@@ -143,7 +142,7 @@ async function upsert(domainObject, location, type, values) {
 
             ++i;
 
-            if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+            if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
             if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
             if (response.statusCode === 422) throw new BoxError(BoxError.BAD_FIELD, response.body.message);
             if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
@@ -202,7 +201,7 @@ async function del(domainObject, location, type, values) {
             .retry(5)
             .ok(() => true));
 
-        if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+        if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
         if (response.statusCode === 404) return;
         if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
         if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));

src/dns/inwx.js

@@ -0,0 +1,217 @@
+'use strict';
+
+exports = module.exports = {
+    removePrivateFields,
+    injectPrivateFields,
+    upsert,
+    get,
+    del,
+    wait,
+    verifyDomainConfig
+};
+
+const { ApiClient, Language } = require('domrobot-client'),
+    assert = require('assert'),
+    BoxError = require('../boxerror.js'),
+    constants = require('../constants.js'),
+    debug = require('debug')('box:dns/inwx'),
+    dig = require('../dig.js'),
+    dns = require('../dns.js'),
+    safe = require('safetydance'),
+    waitForDns = require('./waitfordns.js');
+
+function formatError(response) {
+    return `INWX Api error error [Code: [${response.code}]  Message: ${response.msg}`;
+}
+
+function removePrivateFields(domainObject) {
+    domainObject.config.password = constants.SECRET_PLACEHOLDER;
+    return domainObject;
+}
+
+function injectPrivateFields(newConfig, currentConfig) {
+    if (newConfig.password === constants.SECRET_PLACEHOLDER) newConfig.password = currentConfig.password;
+}
+
+// https://www.inwx.com/en/help/apidoc/f/ch04.html
+function translateError(response) {
+    if (response.code === 2200 || response.code === 2201 || response.code === 2202) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
+    if (response.code === 2003 || response.code === 2004 || response.code === 2005) throw new BoxError(BoxError.BAD_FIELD, formatError(response));
+    if (response.code !== 1000) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
+}
+
+async function login(domainConfig) {
+    const apiClient = new ApiClient(ApiClient.API_URL_LIVE, Language.EN, false /* debug mode */);
+
+    const sharedSecret = ''; // 2FA
+    const [error, response] = await safe(apiClient.login(domainConfig.username, domainConfig.password, sharedSecret));
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
+    if (response.code !== 1000) throw new BoxError(BoxError. ACCESS_DENIED, `Api login error. Code: ${response.code}  Message: ${response.msg}`);
+
+    return apiClient;
+}
+
+async function getDnsRecords(domainConfig, apiClient, zoneName, fqdn, type) {
+    assert.strictEqual(typeof domainConfig, 'object');
+    assert.strictEqual(typeof apiClient, 'object');
+    assert.strictEqual(typeof zoneName, 'string');
+    assert.strictEqual(typeof fqdn, 'string');
+    assert.strictEqual(typeof type, 'string');
+
+    debug(`getDnsRecords: ${fqdn} in zone ${zoneName} of type ${type}`);
+
+    const [error, response] = await safe(apiClient.callApi('nameserver.info', { domain: zoneName, name: fqdn, type }));
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
+    if (response.code !== 1000) throw translateError(response);
+
+    return response.resData.record || []; // 'record' property will be missing if no records
+}
+
+async function upsert(domainObject, location, type, values) {
+    assert.strictEqual(typeof domainObject, 'object');
+    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert(Array.isArray(values));
+
+    const domainConfig = domainObject.config,
+        zoneName = domainObject.zoneName;
+
+    debug(`upsert: ${location} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
+
+    const apiClient = await login(domainConfig);
+    const fqdn = dns.fqdn(location, domainObject.domain);
+    const records = await getDnsRecords(domainConfig, apiClient, zoneName, fqdn, type);
+
+    let i = 0; // // used to track available records to update instead of create
+
+    for (let value of values) {
+        let priority = 0;
+
+        if (type === 'MX') {
+            priority = parseInt(value.split(' ')[0], 10);
+            value = value.split(' ')[1];
+        }
+
+        if (i >= records.length) { // create a new record
+            const data = {
+                type,
+                name: fqdn,
+                domain: zoneName,
+                content: value,
+                prio: priority,
+                ttl: 300  // 300 to 2764800
+            };
+            const [error, response] = await safe(apiClient.callApi('nameserver.createRecord', data));
+            if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
+            if (response.code !== 1000) throw translateError(response);
+        } else { // replace existing record
+            const data = {
+                id: records[i].id,
+                type,
+                name: fqdn,
+                content: value,
+            };
+            const [error, response] = await safe(apiClient.callApi('nameserver.updateRecord', data));
+            if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
+            if (response.code !== 1000) throw translateError(response);
+            ++i; // increment, as we have consumed the record
+        }
+    }
+
+    for (let j = values.length + 1; j < records.length; j++) {
+        const [error, response] = await safe(apiClient.callApi('nameserver.deleteRecord', { id: records[j].id }));
+        if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
+        if (response.code !== 1000) throw translateError(response);
+    }
+}
+
+async function get(domainObject, location, type) {
+    assert.strictEqual(typeof domainObject, 'object');
+    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof type, 'string');
+
+    const domainConfig = domainObject.config,
+        zoneName = domainObject.zoneName;
+
+    const apiClient = await login(domainConfig);
+    const fqdn = dns.fqdn(location, domainObject.domain);
+    const result = await getDnsRecords(domainConfig, apiClient, zoneName, fqdn, type);
+    const tmp = result.map(function (record) { return record.content; });
+    return tmp;
+}
+
+async function del(domainObject, location, type, values) {
+    assert.strictEqual(typeof domainObject, 'object');
+    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert(Array.isArray(values));
+
+    const domainConfig = domainObject.config,
+        zoneName = domainObject.zoneName;
+
+    debug(`del: ${location} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
+
+    const apiClient = await login(domainConfig);
+    const fqdn = dns.fqdn(location, domainObject.domain);
+    const result = await getDnsRecords(domainConfig, apiClient, zoneName, fqdn, type);
+    if (result.length === 0) return;
+
+    const tmp = result.filter(function (record) { return values.some(function (value) { return value === record.content; }); });
+    debug('del: %j', tmp);
+
+    for (const r of tmp) {
+        const [error, response] = await safe(apiClient.callApi('nameserver.deleteRecord', { id: r.id }));
+        if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
+        if (response.code !== 1000) throw translateError(response);
+    }
+}
+
+async function wait(domainObject, subdomain, type, value, options) {
+    assert.strictEqual(typeof domainObject, 'object');
+    assert.strictEqual(typeof subdomain, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert.strictEqual(typeof value, 'string');
+    assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
+
+    const fqdn = dns.fqdn(subdomain, domainObject.domain);
+
+    await waitForDns(fqdn, domainObject.zoneName, type, value, options);
+}
+
+async function verifyDomainConfig(domainObject) {
+    assert.strictEqual(typeof domainObject, 'object');
+
+    const domainConfig = domainObject.config,
+        zoneName = domainObject.zoneName;
+
+    if (typeof domainConfig.username !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'username must be a string');
+    if (typeof domainConfig.password !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'password must be a string');
+
+    const credentials = {
+        username: domainConfig.username,
+        password: domainConfig.password
+    };
+
+    const ip = '127.0.0.1';
+
+    if (constants.TEST) return credentials; // this shouldn't be here
+
+    const [error, nameservers] = await safe(dig.resolve(zoneName, 'NS', { timeout: 5000 }));
+    if (error && error.code === 'ENOTFOUND') throw new BoxError(BoxError.BAD_FIELD, 'Unable to resolve nameservers for this domain');
+    if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers');
+
+    if (!nameservers.every(function (n) { return n.toLowerCase().search(/inwx|xnameserver|domrobot/) !== -1; })) {
+        debug('verifyDomainConfig: %j does not contain INWX NS', nameservers);
+        throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to INWX');
+    }
+
+    const location = 'cloudrontestdns';
+
+    await upsert(domainObject, location, 'A', [ ip ]);
+    debug('verifyDomainConfig: Test A record added');
+
+    await del(domainObject, location, 'A', [ ip ]);
+    debug('verifyDomainConfig: Test A record removed again');
+
+    return credentials;
+}

src/dns/linode.js

@@ -47,7 +47,7 @@ async function getZoneId(domainConfig, zoneName) {
         .retry(5)
         .ok(() => true));
 
-    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
     if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
     if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
 
@@ -84,7 +84,7 @@ async function getZoneRecords(domainConfig, zoneName, name, type) {
             .retry(5)
             .ok(() => true));
 
-        if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+        if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
         if (response.statusCode === 404) throw new BoxError(BoxError.NOT_FOUND, formatError(response));
         if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
         if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
@@ -153,7 +153,7 @@ async function upsert(domainObject, location, type, values) {
                 .retry(5)
                 .ok(() => true));
 
-            if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+            if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
             if (response.statusCode === 400) throw new BoxError(BoxError.BAD_FIELD, formatError(response));
             if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
             if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
@@ -167,7 +167,7 @@ async function upsert(domainObject, location, type, values) {
                 .retry(5)
                 .ok(() => true));
 
-            if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+            if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
             if (response.statusCode === 400) throw new BoxError(BoxError.BAD_FIELD, formatError(response));
             if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
             if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
@@ -211,7 +211,7 @@ async function del(domainObject, location, type, values) {
             .timeout(30 * 1000)
             .retry(5)
             .ok(() => true));
-        if (error && !error.response) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+        if (error && !error.response) throw new BoxError(BoxError.NETWORK_ERROR, error);
         if (response.statusCode === 404) return;
         if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
         if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));

src/dns/namecheap.js

@@ -61,7 +61,7 @@ async function getZone(domainConfig, zoneName) {
     query.TLD = zoneName.slice(query.SLD.length + 1);
 
     const [error, response] = await safe(superagent.get(ENDPOINT).query(query).ok(() => true));
-    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
 
     const parser = new xml2js.Parser();
     const [parserError, result] = await safe(util.promisify(parser.parseString)(response.text));

src/dns/namecom.js

@@ -55,7 +55,7 @@ async function addRecord(domainConfig, zoneName, name, type, values) {
         data.answer = values[0].split(' ')[1];
     } else if (type === 'TXT') {
         // we have to strip the quoting for some odd reason for name.com! If you change that also change updateRecord
-        let tmp = values[0];
+        const tmp = values[0];
         data.answer = tmp.indexOf('"') === 0 && tmp.lastIndexOf('"') === tmp.length-1 ? tmp.slice(1, tmp.length-1) : tmp;
     } else {
         data.answer = values[0];
@@ -67,7 +67,7 @@ async function addRecord(domainConfig, zoneName, name, type, values) {
         .send(data)
         .ok(() => true));
 
-    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
     if (response.statusCode === 403) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
     if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
 }
@@ -93,7 +93,7 @@ async function updateRecord(domainConfig, zoneName, recordId, name, type, values
         data.answer = values[0].split(' ')[1];
     } else if (type === 'TXT') {
         // we have to strip the quoting for some odd reason for name.com! If you change that also change addRecord
-        let tmp = values[0];
+        const tmp = values[0];
         data.answer = tmp.indexOf('"') === 0 && tmp.lastIndexOf('"') === tmp.length-1 ? tmp.slice(1, tmp.length-1) : tmp;
     } else {
         data.answer = values[0];
@@ -105,7 +105,7 @@ async function updateRecord(domainConfig, zoneName, recordId, name, type, values
         .send(data)
         .ok(() => true));
 
-    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
     if (response.statusCode === 403) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
     if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
 }
@@ -123,7 +123,7 @@ async function getInternal(domainConfig, zoneName, name, type) {
         .timeout(30 * 1000)
         .ok(() => true));
 
-    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
     if (response.statusCode === 403) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
     if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
 
@@ -194,7 +194,7 @@ async function del(domainObject, location, type, values) {
         .auth(domainConfig.username, domainConfig.token)
         .timeout(30 * 1000)
         .ok(() => true));
-    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
     if (response.statusCode === 403) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
     if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
 }

src/dns/netcup.js

@@ -51,7 +51,7 @@ async function login(domainConfig) {
     };
 
     const [error, response] = await safe(superagent.post(API_ENDPOINT).send(data).ok(() => true));
-    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
     if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
     if (!response.body.responsedata.apisessionid) throw new BoxError(BoxError.ACCESS_DENIED, 'invalid api password');
 
@@ -76,7 +76,7 @@ async function getAllRecords(domainConfig, apiSessionId, zoneName) {
     };
 
     const [error, response] = await safe(superagent.post(API_ENDPOINT).send(data).ok(() => true));
-    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
     if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
 
     return response.body.responsedata.dnsrecords || [];
@@ -98,7 +98,7 @@ async function upsert(domainObject, location, type, values) {
 
     const result = await getAllRecords(domainConfig, apiSessionId, zoneName);
 
-    let records = [];
+    const records = [];
 
     values.forEach(function (value) {
         // remove possible quotation
@@ -134,7 +134,7 @@ async function upsert(domainObject, location, type, values) {
     };
 
     const [error, response] = await safe(superagent.post(API_ENDPOINT).send(data).ok(() => true));
-    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
     if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
     if (response.body.statuscode !== 2000) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
 }
@@ -174,14 +174,14 @@ async function del(domainObject, location, type, values) {
 
     const result = await getAllRecords(domainConfig, apiSessionId, zoneName);
 
-    let records = [];
+    const records = [];
 
     values.forEach(function (value) {
         // remove possible quotation
         if (value.charAt(0) === '"') value = value.slice(1);
         if (value.charAt(value.length -1) === '"') value = value.slice(0, -1);
 
-        let record = result.find(function (r) { return r.hostname === name && r.type === type && r.destination === value; });
+        const record = result.find(function (r) { return r.hostname === name && r.type === type && r.destination === value; });
         if (!record) return;
 
         record.deleterecord = true;
@@ -205,7 +205,7 @@ async function del(domainObject, location, type, values) {
     };
 
     const [error, response] = await safe(superagent.post(API_ENDPOINT).send(data).ok(() => true));
-    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
     if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
     if (response.body.statuscode !== 2000) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
 }

src/dns/porkbun.js

@@ -38,26 +38,30 @@ function injectPrivateFields(newConfig, currentConfig) {
     if (newConfig.secretapikey === constants.SECRET_PLACEHOLDER) newConfig.secretapikey = currentConfig.secretapikey;
 }
 
+async function createRequest(method, url, data) {
+    assert.strictEqual(typeof method, 'string');
+    assert.strictEqual(typeof url, 'string');
+    assert.strictEqual(typeof data, 'object');
+
+    await timers.setTimeout(3000); // see rate limit note at top of file
+    return superagent(method, url).retry(5).timeout(30 * 1000).send(data).ok(() => true);
+}
+
 async function getDnsRecords(domainConfig, zoneName, name, type) {
     assert.strictEqual(typeof domainConfig, 'object');
     assert.strictEqual(typeof zoneName, 'string');
     assert.strictEqual(typeof name, 'string');
     assert.strictEqual(typeof type, 'string');
 
-    debug(`get: ${name} in zone ${zoneName} of type ${type}`);
+    debug(`get: ${name} zone:${zoneName} type:${type}`);
 
     const data = {
         secretapikey: domainConfig.secretapikey,
         apikey: domainConfig.apikey,
     };
 
-    const [error, response] = await safe(superagent.post(`${PORKBUN_API}/retrieveByNameType/${zoneName}/${type}/${name}`)
-        .retry(5)
-        .send(data)
-        .timeout(30 * 1000)
-        .ok(() => true));
-
-    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    const [error, response] = await safe(createRequest('POST', `${PORKBUN_API}/retrieveByNameType/${zoneName}/${type}/${name}`, data));
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
     if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
     if (response.body.status !== 'SUCCESS') throw new BoxError(BoxError.EXTERNAL_ERROR, `Invalid status in response: ${JSON.stringify(response.body)}`);
     if (!Array.isArray(response.body.records)) throw new BoxError(BoxError.EXTERNAL_ERROR, `Invalid records in response: ${JSON.stringify(response.body)}`);
@@ -78,13 +82,8 @@ async function delDnsRecords(domainConfig, zoneName, name, type) {
     };
 
     // deletes all the records matching type+name
-    const [error, response] = await safe(superagent.post(`${PORKBUN_API}/deleteByNameType/${zoneName}/${type}/${name}`)
-        .retry(5)
-        .send(data)
-        .timeout(30 * 1000)
-        .ok(() => true));
-
-    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    const [error, response] = await safe(createRequest('POST', `${PORKBUN_API}/deleteByNameType/${zoneName}/${type}/${name}`, data));
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
     if (response.statusCode === 400) return; // not found, "Could not delete record."
     if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
     if (response.body.status !== 'SUCCESS') throw new BoxError(BoxError.EXTERNAL_ERROR, `Invalid status in response: ${JSON.stringify(response.body)}`);
@@ -100,7 +99,7 @@ async function upsert(domainObject, location, type, values) {
         zoneName = domainObject.zoneName,
         name = dns.getName(domainObject, location, type) || '';
 
-    debug(`upsert: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
+    debug(`upsert: ${name} zone:${zoneName} type:${type} values:${JSON.stringify(values)}`);
 
     await delDnsRecords(domainConfig, zoneName, name, type);
 
@@ -122,20 +121,14 @@ async function upsert(domainObject, location, type, values) {
             data.content = value;
         }
 
-        const [error, response] = await safe(superagent.post(`${PORKBUN_API}/create/${zoneName}`)
-            .retry(5)
-            .timeout(30 * 1000)
-            .send(data)
-            .ok(() => true));
+        const [error, response] = await safe(createRequest('POST', `${PORKBUN_API}/create/${zoneName}`, data));
 
-        if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+        if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
         if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
         if (response.body.status !== 'SUCCESS') throw new BoxError(BoxError.EXTERNAL_ERROR, `Invalid status in response: ${JSON.stringify(response.body)}`);
         if (!response.body.id) throw new BoxError(BoxError.EXTERNAL_ERROR, `Invalid id in response: ${JSON.stringify(response.body)}`);
 
         debug(`upsert: created record with id ${response.body.id}`);
-
-        await timers.setTimeout(1500); // see rate limit note at top of file
     }
 }
 
@@ -162,7 +155,7 @@ async function del(domainObject, location, type, values) {
         zoneName = domainObject.zoneName,
         name = dns.getName(domainObject, location, type) || '';
 
-    debug(`del: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
+    debug(`del: ${name} zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
 
     const data = {
         secretapikey: domainConfig.secretapikey,
@@ -174,18 +167,11 @@ async function del(domainObject, location, type, values) {
     const ids = records.filter(r => values.includes(r.content)).map(r => r.id);
 
     for (const id of ids) {
-        const [error, response] = await safe(superagent.post(`${PORKBUN_API}/delete/${zoneName}/${id}`)
-            .retry(5)
-            .send(data)
-            .timeout(30 * 1000)
-            .ok(() => true));
-
-        if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+        const [error, response] = await safe(createRequest('POST', `${PORKBUN_API}/delete/${zoneName}/${id}`, data));
+        if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
         if (response.statusCode === 400) continue; // not found! "Invalid record id."
         if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
         if (response.body.status !== 'SUCCESS') throw new BoxError(BoxError.EXTERNAL_ERROR, `Invalid status in response: ${JSON.stringify(response.body)}`);
-
-        await timers.setTimeout(1500); // see rate limit note at top of file
     }
 }
 

src/dns/vultr.js

@@ -44,14 +44,14 @@ async function getZoneRecords(domainConfig, zoneName, name, type) {
 
     debug(`getInternal: getting dns records of ${zoneName} with ${name} and type ${type}`);
 
-    let per_page = 100, cursor = null;
-    let records = [];
+    const per_page = 100;
+    let cursor = null, records = [];
 
     do {
         const url = `${VULTR_ENDPOINT}/domains/${zoneName}/records?per_page=${per_page}` + (cursor ? `&cursor=${cursor}` : '');
 
         const [error, response] = await safe(superagent.get(url).set('Authorization', 'Bearer ' + domainConfig.token).timeout(30 * 1000).retry(5).ok(() => true));
-        if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+        if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
         if (response.statusCode === 404) throw new BoxError(BoxError.NOT_FOUND, formatError(response));
         if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
         if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
@@ -94,7 +94,8 @@ async function upsert(domainObject, location, type, values) {
 
     const records = await getZoneRecords(domainConfig, zoneName, name, type);
 
-    let i = 0, recordIds = []; // used to track available records to update instead of create
+    let i = 0;
+    const recordIds = []; // used to track available records to update instead of create
 
     for (const value of values) {
         const data = {
@@ -121,7 +122,7 @@ async function upsert(domainObject, location, type, values) {
                 .retry(5)
                 .ok(() => true));
 
-            if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+            if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
             if (response.statusCode === 400) throw new BoxError(BoxError.BAD_FIELD, formatError(response));
             if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
             if (response.statusCode !== 201) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
@@ -137,7 +138,7 @@ async function upsert(domainObject, location, type, values) {
 
             ++i;
 
-            if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+            if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
             if (response.statusCode === 400) throw new BoxError(BoxError.BAD_FIELD, formatError(response));
             if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
             if (response.statusCode !== 204) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
@@ -181,7 +182,7 @@ async function del(domainObject, location, type, values) {
             .retry(5)
             .ok(() => true));
 
-        if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+        if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
         if (response.statusCode === 404) continue;
         if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
         if (response.statusCode !== 204) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));

src/docker.js

@@ -26,7 +26,7 @@ exports = module.exports = {
 
     update,
 
-    parseImageName,
+    parseImageRef,
 
     createExec,
     startExec,
@@ -47,7 +47,6 @@ const apps = require('./apps.js'),
     promiseRetry = require('./promise-retry.js'),
     services = require('./services.js'),
     settings = require('./settings.js'),
-    semver = require('semver'),
     shell = require('./shell.js')('docker'),
     safe = require('safetydance'),
     timers = require('timers/promises'),
@@ -77,6 +76,27 @@ function removePrivateFields(registryConfig) {
     return registryConfig;
 }
 
+function parseImageRef(imageRef) {
+    assert.strictEqual(typeof imageRef, 'string');
+
+    // a ref is like registry.docker.com/cloudron/base:4.2.0@sha256:46da2fffb36353ef714f97ae8e962bd2c212ca091108d768ba473078319a47f4
+    // registry.docker.com is registry name . cloudron is namespace . base is image name . cloudron/base is repository path
+    // registry.docker.com/cloudron/base is fullRepositoryName
+    const result = { fullRepositoryName: null, registry: null, tag: null, digest: null };
+    result.fullRepositoryName = imageRef.split(/[:@]/)[0];
+    const parts = result.fullRepositoryName.split('/');
+    result.registry = parts.length === 3 ? parts[0] : null;
+    let remaining = imageRef.substr(result.fullRepositoryName.length);
+    if (remaining.startsWith(':')) {
+        result.tag = remaining.substr(1).split('@', 1)[0];
+        remaining = remaining.substr(result.tag.length + 1); // also ':'
+    }
+
+    if (remaining.startsWith('@sha256:')) result.digest = remaining.substr(8);
+
+    return result;
+}
+
 async function ping() {
     // do not let the request linger
     const connection = new Docker({ socketPath: DOCKER_SOCKET_PATH, timeout: 1000 });
@@ -89,12 +109,21 @@ async function ping() {
     throw new BoxError(BoxError.DOCKER_ERROR, 'Unable to ping the docker daemon');
 }
 
-async function getAuthConfig(image) {
-    // https://github.com/docker/distribution/blob/release/2.7/reference/normalize.go#L62
-    const parts = image.split('/');
-    if (parts.length === 1 || (parts[0].match(/[.:]/) === null)) return null; // public docker registry
+async function getAuthConfig(imageRef) {
+    assert.strictEqual(typeof imageRef, 'string');
+
+    const parsedRef = parseImageRef(imageRef);
+
+    // images in our cloudron namespace are always unauthenticated to not interfere with any user limits
+    if (parsedRef.registry === null && parsedRef.fullRepositoryName.startsWith('cloudron/')) return null;
 
     const registryConfig = await getRegistryConfig();
+    if (registryConfig.provider === 'noop') return null;
+
+    if (registryConfig.serverAddress !== parsedRef.registry) { // ideally they match but there's too many docker registry domains!
+        if (!registryConfig.serverAddress.includes('.docker.')) return null;
+        if (parsedRef.registry !== null && !parsedRef.includes('.docker.')) return null;
+    }
 
     // https://github.com/apocas/dockerode#pull-from-private-repos
     const autoConfig = {
@@ -108,14 +137,18 @@ async function getAuthConfig(image) {
     return autoConfig;
 }
 
-async function pullImage(manifest) {
-    const authConfig = await getAuthConfig(manifest.dockerImage);
+async function pullImage(imageRef) {
+    assert.strictEqual(typeof imageRef, 'string');
 
-    debug(`pullImage: will pull ${manifest.dockerImage}. auth: ${authConfig ? 'yes' : 'no'}`);
+    const authConfig = await getAuthConfig(imageRef);
 
-    const [error, stream] = await safe(gConnection.pull(manifest.dockerImage, { authconfig: authConfig }));
-    if (error && error.statusCode === 404) throw new BoxError(BoxError.NOT_FOUND, `Unable to pull image ${manifest.dockerImage}. message: ${error.message} statusCode: ${error.statusCode}`);
-    if (error) throw new BoxError(BoxError.DOCKER_ERROR, `Unable to pull image ${manifest.dockerImage}. Please check the network or if the image needs authentication. statusCode: ${error.statusCode}`);
+    debug(`pullImage: will pull ${imageRef}. auth: ${authConfig ? 'yes' : 'no'}`);
+
+    const [error, stream] = await safe(gConnection.pull(imageRef, { authconfig: authConfig }));
+    if (error && error.statusCode === 404) throw new BoxError(BoxError.NOT_FOUND, `Unable to pull image ${imageRef}. message: ${error.message} statusCode: ${error.statusCode}`);
+    // toomanyrequests is flagged as a 500. dockerhub appears to have 10 pulls her hour per IP limit
+    if (error && error.statusCode === 500) throw new BoxError(BoxError.DOCKER_ERROR, `Unable to pull image ${imageRef}. registry error: ${JSON.stringify(error)}`);
+    if (error) throw new BoxError(BoxError.DOCKER_ERROR, `Unable to pull image ${imageRef}. Please check the network or if the image needs authentication. statusCode: ${error.statusCode}`);
 
     return new Promise((resolve, reject) => {
         // https://github.com/dotcloud/docker/issues/1074 says each status message is emitted as a chunk
@@ -126,13 +159,13 @@ async function pullImage(manifest) {
 
             // The data.status here is useless because this is per layer as opposed to per image
             if (!data.status && data.error) { // data is { errorDetail: { message: xx } , error: xx }
-                debug(`pullImage error ${manifest.dockerImage}: ${data.errorDetail.message}`);
+                debug(`pullImage error ${imageRef}: ${data.errorDetail.message}`);
                 layerError = data.errorDetail;
             }
         });
 
         stream.on('end', function () {
-            debug(`downloaded image ${manifest.dockerImage} . error: ${!!layerError}`);
+            debug(`downloaded image ${imageRef} . error: ${!!layerError}`);
 
             if (!layerError) return resolve();
 
@@ -140,7 +173,7 @@ async function pullImage(manifest) {
         });
 
         stream.on('error', function (error) { // this is only hit for stream error and not for some download error
-            debug('error pulling image %s: %o', manifest.dockerImage, error);
+            debug(`error pulling image ${imageRef}: %o`, error);
             reject(new BoxError(BoxError.DOCKER_ERROR, error.message));
         });
     });
@@ -149,15 +182,32 @@ async function pullImage(manifest) {
 async function downloadImage(manifest) {
     assert.strictEqual(typeof manifest, 'object');
 
-    debug(`downloadImage ${manifest.dockerImage}`);
+    debug(`downloadImage: ${manifest.dockerImage}`);
 
     const image = gConnection.getImage(manifest.dockerImage);
 
     const [error, result] = await safe(image.inspect());
     if (!error && result) return; // image is already present locally
 
-    await promiseRetry({ times: 10, interval: 5000, debug, retry: (pullError) => pullError.reason !== BoxError.NOT_FOUND && pullError.reason !== BoxError.FS_ERROR }, async () => {
-        await pullImage(manifest);
+    const parsedManifestRef = parseImageRef(manifest.dockerImage);
+
+    await promiseRetry({ times: 10, interval: 5000, debug, retry: (pullError) => pullError.reason !== BoxError.FS_ERROR }, async () => {
+        if (parsedManifestRef.registry !== null || !parsedManifestRef.fullRepositoryName.startsWith('cloudron/')) return await pullImage(manifest.dockerImage);
+
+        let upstreamRef = null;
+        for (const registry of [ 'registry.docker.com', 'registry.ipv4.docker.com', 'quay.io' ]) {
+            upstreamRef = `${registry}/${manifest.dockerImage}`;
+            const [pullError] = await safe(pullImage(upstreamRef));
+            if (!pullError) break;
+        }
+
+        if (!upstreamRef) throw new BoxError(BoxError.DOCKER_ERROR, `Unable to pull image ${manifest.dockerImage} from dockerhub or quay`);
+
+        // retag the downloaded image to not have the registry name. this prevents 'docker run' from redownloading it
+        debug(`downloadImage: tagging ${upstreamRef} as ${parsedManifestRef.fullRepositoryName}:${parsedManifestRef.tag}`);
+        await gConnection.getImage(upstreamRef).tag({ repo: parsedManifestRef.fullRepositoryName, tag: parsedManifestRef.tag });
+        debug(`downloadImage: untagging ${upstreamRef}`);
+        await deleteImage(upstreamRef);
     });
 }
 
@@ -406,10 +456,17 @@ async function createSubcontainer(app, name, cmd, options) {
     if (capabilities.includes('mlock')) containerOptions.HostConfig.CapAdd.push('IPC_LOCK'); // mlock prevents swapping
     if (!capabilities.includes('ping')) containerOptions.HostConfig.CapDrop.push('NET_RAW'); // NET_RAW is included by default by Docker
 
+    containerOptions.HostConfig.Devices = Object.keys(app.devices).map((d) => {
+        if (!safe.fs.existsSync(d)) {
+            debug(`createSubcontainer: device ${d} does not exist. Skipping...`);
+            return null;
+        }
+
+        return { PathOnHost: d, PathInContainer: d, CgroupPermissions: 'rwm' };
+    }).filter(d => d);
+
     if (capabilities.includes('vaapi') && safe.fs.existsSync('/dev/dri')) {
-        containerOptions.HostConfig.Devices = [
-            { PathOnHost: '/dev/dri', PathInContainer: '/dev/dri', CgroupPermissions: 'rwm' }
-        ];
+        containerOptions.HostConfig.Devices.push({ PathOnHost: '/dev/dri', PathInContainer: '/dev/dri', CgroupPermissions: 'rwm' });
     }
 
     const mergedOptions = Object.assign({}, containerOptions, options);
@@ -515,12 +572,11 @@ async function stopContainers(appId) {
     }
 }
 
-async function deleteImage(manifest) {
-    assert(!manifest || typeof manifest === 'object');
+async function deleteImage(imageRef) {
+    assert.strictEqual(typeof imageRef, 'string');
 
-    const dockerImage = manifest ? manifest.dockerImage : null;
-    if (!dockerImage) return;
-    if (dockerImage.includes('//') || dockerImage.startsWith('/')) return; // a common mistake is to paste a https:// as docker image. this results in a crash at runtime in dockerode module (https://github.com/apocas/dockerode/issues/548)
+    if (!imageRef) return;
+    if (imageRef.includes('//') || imageRef.startsWith('/')) return; // a common mistake is to paste a https:// as docker image. this results in a crash at runtime in dockerode module (https://github.com/apocas/dockerode/issues/548)
 
     const removeOptions = {
         force: false, // might be shared with another instance of this app
@@ -530,13 +586,14 @@ async function deleteImage(manifest) {
     // registry v1 used to pull down all *tags*. this meant that deleting image by tag was not enough (since that
     // just removes the tag). we used to remove the image by id. this is not required anymore because aliases are
     // not created anymore after https://github.com/docker/docker/pull/10571
-    const [error] = await safe(gConnection.getImage(dockerImage).remove(removeOptions));
+    debug(`deleteImage: removing ${imageRef}`);
+    const [error] = await safe(gConnection.getImage(imageRef.replace(/@sha256:.*/,'')).remove(removeOptions)); // can't have the manifest id. won't remove anythin
     if (error && error.statusCode === 400) return; // invalid image format. this can happen if user installed with a bad --docker-image
     if (error && error.statusCode === 404) return; // not found
     if (error && error.statusCode === 409) return; // another container using the image
 
     if (error) {
-        debug('Error removing image %s : %o', dockerImage, error);
+        debug(`Error removing image ${imageRef} : %o`, error);
         throw new BoxError(BoxError.DOCKER_ERROR, error);
     }
 }
@@ -674,11 +731,3 @@ async function setRegistryConfig(registryConfig) {
 
     await settings.setJson(settings.REGISTRY_CONFIG_KEY, registryConfig);
 }
-
-function parseImageName(imageName) {
-    const repository = imageName.split(':', 1)[0];
-    const tag = imageName.substr(repository.length + 1).split('@', 1)[0];
-    const digest = imageName.substr(repository.length + 1 + tag.length + 1).split(':', 2)[1];
-
-    return { repository, tag, version: semver.parse(tag), digest };
-}

src/domains.js

@@ -62,6 +62,7 @@ function api(provider) {
     case 'gandi': return require('./dns/gandi.js');
     case 'godaddy': return require('./dns/godaddy.js');
     case 'hetzner': return require('./dns/hetzner.js');
+    case 'inwx': return require('./dns/inwx.js');
     case 'linode': return require('./dns/linode.js');
     case 'vultr': return require('./dns/vultr.js');
     case 'namecom': return require('./dns/namecom.js');

src/eventlog.js

@@ -30,11 +30,18 @@ exports = module.exports = {
     ACTION_APP_STOP: 'app.stop',
     ACTION_APP_RESTART: 'app.restart',
 
+    ACTION_ARCHIVES_ADD: 'archives.add',
+    ACTION_ARCHIVES_DEL: 'archives.del',
+
     ACTION_BACKUP_FINISH: 'backup.finish',
     ACTION_BACKUP_START: 'backup.start',
     ACTION_BACKUP_CLEANUP_START: 'backup.cleanup.start', // obsolete
     ACTION_BACKUP_CLEANUP_FINISH: 'backup.cleanup.finish',
 
+    ACTION_BRANDING_NAME: 'branding.name',
+    ACTION_BRANDING_FOOTER: 'branding.footer',
+    ACTION_BRANDING_AVATAR: 'branding.avatar',
+
     ACTION_CERTIFICATE_NEW: 'certificate.new',
     ACTION_CERTIFICATE_RENEWAL: 'certificate.renew', // obsolete
     ACTION_CERTIFICATE_CLEANUP: 'certificate.cleanup',
@@ -49,6 +56,11 @@ exports = module.exports = {
 
     ACTION_EXTERNAL_LDAP_CONFIGURE: 'externalldap.configure',
 
+    ACTION_GROUP_ADD: 'group.add',
+    ACTION_GROUP_REMOVE: 'group.remove',
+    ACTION_GROUP_UPDATE: 'group.update',
+    ACTION_GROUP_MEMBERSHIP: 'group.membership',
+
     ACTION_INSTALL_FINISH: 'cloudron.install.finish',
 
     ACTION_MAIL_LOCATION: 'mail.location',

src/externalldap.js

@@ -85,8 +85,8 @@ async function setConfig(newConfig, auditSource) {
     await settings.setJson(settings.EXTERNAL_LDAP_KEY, newConfig);
 
     if (newConfig.provider === 'noop') {
-        await users.resetSource(); // otherwise, the owner could be 'ldap' source and lock themselves out
-        await groups.resetSource();
+        await users.resetSources(); // otherwise, the owner could be 'ldap' source and lock themselves out
+        await groups.resetSources();
     }
 
     await eventlog.add(eventlog.ACTION_EXTERNAL_LDAP_CONFIGURE, auditSource, { oldConfig: removePrivateFields(currentConfig), config: removePrivateFields(newConfig) });
@@ -416,7 +416,7 @@ async function syncGroups(config, progressCallback) {
 
         if (!result) {
             debug(`syncGroups: [adding group] groupname=${groupName}`);
-            const [error] = await safe(groups.add({ name: groupName, source: 'ldap' }));
+            const [error] = await safe(groups.add({ name: groupName, source: 'ldap' }, AuditSource.EXTERNAL_LDAP));
             if (error) debug('syncGroups: Failed to create group', groupName, error);
         } else {
             // convert local group to ldap group. 2 reasons:
@@ -492,7 +492,7 @@ async function syncGroupMembers(config, progressCallback) {
 
             userIds.push(userObject.id);
         }
-        const [setError] = await safe(groups.setMembers(group, userIds, { skipSourceCheck: true }));
+        const [setError] = await safe(groups.setMembers(group, userIds, { skipSourceCheck: true }, AuditSource.EXTERNAL_LDAP));
         if (setError) debug(`syncGroupMembers: Failed to set members of group ${group.name}. %o`, setError);
     }
 

src/graphs.js

@@ -62,7 +62,7 @@ async function getContainerStats(name, fromMinutes, noNullPoints) {
         };
 
         const [error, response] = await safe(superagent.get(graphiteUrl).query(query).timeout(30 * 1000).ok(() => true));
-        if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+        if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
         if (response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, `Unknown error with ${target}: ${response.status} ${response.text}`);
 
         results.push(response.body[0] && response.body[0].datapoints ? response.body[0].datapoints : []);
@@ -102,7 +102,7 @@ async function getSystem(fromMinutes, noNullPoints) {
     };
 
     const [memCpuError, memCpuResponse] = await safe(superagent.get(graphiteUrl).query(query).timeout(30 * 1000).ok(() => true));
-    if (memCpuError) throw new BoxError(BoxError.NETWORK_ERROR, memCpuError.message);
+    if (memCpuError) throw new BoxError(BoxError.NETWORK_ERROR, memCpuError);
     if (memCpuResponse.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, `Unknown error: ${memCpuResponse.status} ${memCpuResponse.text}`);
 
     const appResponses = {};

src/groups.js

@@ -2,7 +2,7 @@
 
 exports = module.exports = {
     add,
-    remove,
+    del,
     get,
     getByName,
 
@@ -13,13 +13,12 @@ exports = module.exports = {
     list,
     listWithMembers,
 
-    getMembers,
+    getMemberIds,
     setMembers,
-    removeMember,
     isMember,
 
     setLocalMembership,
-    resetSource,
+    resetSources,
 
     // exported for testing
     _getMembership: getMembership
@@ -29,6 +28,7 @@ const assert = require('assert'),
     BoxError = require('./boxerror.js'),
     constants = require('./constants.js'),
     database = require('./database.js'),
+    eventlog = require('./eventlog.js'),
     safe = require('safetydance'),
     uuid = require('uuid');
 
@@ -57,8 +57,9 @@ function validateSource(source) {
     return null;
 }
 
-async function add(group) {
+async function add(group, auditSource) {
     assert.strictEqual(typeof group, 'object');
+    assert(auditSource && typeof auditSource === 'object');
 
     let { name, source } = group;
 
@@ -77,19 +78,24 @@ async function add(group) {
     if (error && error.code === 'ER_DUP_ENTRY') throw new BoxError(BoxError.ALREADY_EXISTS, error);
     if (error) throw error;
 
+    await eventlog.add(eventlog.ACTION_GROUP_ADD, auditSource, { id, name, source });
+
     return { id, name, source };
 }
 
-async function remove(id) {
-    assert.strictEqual(typeof id, 'string');
+async function del(group, auditSource) {
+    assert.strictEqual(typeof group, 'object');
+    assert(auditSource && typeof auditSource === 'object');
 
-    // also cleanup the groupMembers table
-    let queries = [];
-    queries.push({ query: 'DELETE FROM groupMembers WHERE groupId = ?', args: [ id ] });
-    queries.push({ query: 'DELETE FROM userGroups WHERE id = ?', args: [ id ] });
+    const queries = [
+        { query: 'DELETE FROM groupMembers WHERE groupId = ?', args: [ group.id ] },
+        { query: 'DELETE FROM userGroups WHERE id = ?', args: [ group.id ] }
+    ];
 
     const result = await database.transaction(queries);
     if (result[1].affectedRows !== 1) throw new BoxError(BoxError.NOT_FOUND, 'Group not found');
+
+    await eventlog.add(eventlog.ACTION_GROUP_REMOVE, auditSource, { group });
 }
 
 async function get(id) {
@@ -140,7 +146,7 @@ async function listWithMembers() {
     return results;
 }
 
-async function getMembers(groupId) {
+async function getMemberIds(groupId) {
     assert.strictEqual(typeof groupId, 'string');
 
     const result = await database.query('SELECT userId FROM groupMembers WHERE groupId=?', [ groupId ]);
@@ -180,10 +186,11 @@ async function setLocalMembership(user, localGroupIds) {
     if (error) throw error;
 }
 
-async function setMembers(group, userIds, options) {
+async function setMembers(group, userIds, options, auditSource) {
     assert.strictEqual(typeof group, 'object');
     assert(Array.isArray(userIds));
     assert.strictEqual(typeof options, 'object');
+    assert(auditSource && typeof auditSource === 'object');
 
     if (!options.skipSourceCheck && group.source === 'ldap') throw new BoxError(BoxError.BAD_STATE, 'Cannot set members of external group');
 
@@ -197,14 +204,8 @@ async function setMembers(group, userIds, options) {
     if (error && error.code === 'ER_NO_REFERENCED_ROW_2') throw new BoxError(BoxError.NOT_FOUND, 'Group not found');
     if (error && error.code === 'ER_DUP_ENTRY') throw new BoxError(BoxError.CONFLICT, 'Duplicate member in list');
     if (error) throw error;
-}
 
-async function removeMember(groupId, userId) {
-    assert.strictEqual(typeof groupId, 'string');
-    assert.strictEqual(typeof userId, 'string');
-
-    const result = await database.query('DELETE FROM groupMembers WHERE groupId = ? AND userId = ?', [ groupId, userId ]);
-    if (result.affectedRows !== 1) throw new BoxError(BoxError.NOT_FOUND, 'Group not found');
+    await eventlog.add(eventlog.ACTION_GROUP_MEMBERSHIP, auditSource, { group, userIds });
 }
 
 async function isMember(groupId, userId) {
@@ -248,15 +249,18 @@ async function update(id, data) {
     if (result.affectedRows !== 1) throw new BoxError(BoxError.NOT_FOUND, 'Group not found');
 }
 
-async function setName(group, name) {
+async function setName(group, name, auditSource) {
     assert.strictEqual(typeof group, 'object');
     assert.strictEqual(typeof name, 'string');
+    assert(auditSource && typeof auditSource === 'object');
 
     if (group.source === 'ldap') throw new BoxError(BoxError.BAD_STATE, 'Cannot set name of external group');
 
     await update(group.id, { name });
+
+    await eventlog.add(eventlog.ACTION_GROUP_UPDATE, auditSource, { oldName: group.name, group });
 }
 
-async function resetSource() {
+async function resetSources() {
     await database.query('UPDATE userGroups SET source = ?', [ '' ]);
 }

src/infra_version.js

@@ -9,14 +9,14 @@ exports = module.exports = {
     'version': '49.8.0',
 
     // a major version bump in the db containers will trigger the restore logic that uses the db dumps
-    // docker inspect --format='{{index .RepoDigests 0}}' $IMAGE to get the sha256
+    // docker inspect --format='{{index .RepoDigests 0}}' $IMAGE to get the sha256 . note this has registry in it because manifest id is registry specific!
     'images': {
-        'base': 'registry.docker.com/cloudron/base:4.2.0@sha256:46da2fffb36353ef714f97ae8e962bd2c212ca091108d768ba473078319a47f4',
+        // 'base': 'registry.docker.com/cloudron/base:4.2.0@sha256:46da2fffb36353ef714f97ae8e962bd2c212ca091108d768ba473078319a47f4',
         'graphite': 'registry.docker.com/cloudron/graphite:3.4.3@sha256:75df420ece34b31a7ce8d45b932246b7f524c123e1854f5e8f115a9e94e33f20',
-        'mail': 'registry.docker.com/cloudron/mail:3.13.1@sha256:1ebc59926b42dca2b6803728c2902b98ebf023944dffef5345fa954b022a5774',
+        'mail': 'registry.docker.com/cloudron/mail:3.14.2@sha256:b760d8476194aff96050d48f856c283584fb7886ac3628a17c48811c22c8836d',
         'mongodb': 'registry.docker.com/cloudron/mongodb:6.0.0@sha256:1108319805acfb66115aa96a8fdbf2cded28d46da0e04d171a87ec734b453d1e',
         'mysql': 'registry.docker.com/cloudron/mysql:3.4.3@sha256:8934c5ddcd69f24740d9a38f0de2937e47240238f3b8f5c482862eeccc5a21d2',
-        'postgresql': 'registry.docker.com/cloudron/postgresql:5.2.3@sha256:9b7d5147e9c8008e4766cc80ebf4b833f3dfcf19ef0d81b013dfab76995d8d16',
+        'postgresql': 'registry.docker.com/cloudron/postgresql:5.3.1@sha256:eaea598aec086c90c0bb7bb8227bcde51b368bcca83d0082a4919bbb6f2d039f',
         'redis': 'registry.docker.com/cloudron/redis:3.5.4@sha256:7c97adb4ee1606d5a0d38aa5ed107a9c27efa13a251f1c1585979292c23de4ec',
         'sftp': 'registry.docker.com/cloudron/sftp:3.8.9@sha256:f2a126839df99ca420a3ad8177594f58b113f6292e98719f2cf2e0ddc3597696',
         'turn': 'registry.docker.com/cloudron/turn:1.7.2@sha256:9ed8da613c1edc5cb8700657cf6e49f0f285b446222a8f459f80919945352f6d',

src/ldapserver.js

@@ -460,7 +460,7 @@ async function verifyMailboxPassword(mailbox, password) {
     if (mailbox.ownerType === mail.OWNERTYPE_USER) {
         return await users.verify(mailbox.ownerId, password, users.AP_MAIL /* identifier */, { skipTotpCheck: true });
     } else if (mailbox.ownerType === mail.OWNERTYPE_GROUP) {
-        const userIds = await groups.getMembers(mailbox.ownerId);
+        const userIds = await groups.getMemberIds(mailbox.ownerId);
 
         let verifiedUser = null;
         for (const userId of userIds) {

src/locker.js

@@ -1,75 +0,0 @@
-'use strict';
-
-const assert = require('assert'),
-    BoxError = require('./boxerror.js'),
-    debug = require('debug')('box:locker'),
-    EventEmitter = require('events').EventEmitter,
-    util = require('util');
-
-function Locker() {
-    this._operation = null;
-    this._timestamp = null;
-    this._watcherId = -1;
-    this._lockDepth = 0; // recursive locks
-}
-util.inherits(Locker, EventEmitter);
-
-// these are mutually exclusive operations
-Locker.prototype.OP_BOX_UPDATE = 'box_update';
-Locker.prototype.OP_INFRA_START = 'infra_start';
-Locker.prototype.OP_FULL_BACKUP = 'full_backup';
-Locker.prototype.OP_APPTASK = 'apptask';
-
-Locker.prototype.lock = function (operation) {
-    assert.strictEqual(typeof operation, 'string');
-
-    if (this._operation !== null) {
-        let error = new BoxError(BoxError.CONFLICT, `Locked for ${this._operation}`);
-        error.operation = this._operation;
-        return error;
-    }
-
-    this._operation = operation;
-    ++this._lockDepth;
-    this._timestamp = new Date();
-    this._watcherId = setInterval(() => { debug('Lock unreleased %s', this._operation); }, 1000 * 60 * 5);
-
-    debug('Acquired : %s', this._operation);
-
-    this.emit('locked', this._operation);
-
-    return null;
-};
-
-Locker.prototype.recursiveLock = function (operation) {
-    if (this._operation === operation) {
-        ++this._lockDepth;
-        debug('Re-acquired : %s Depth : %s', this._operation, this._lockDepth);
-        return null;
-    }
-
-    return this.lock(operation);
-};
-
-Locker.prototype.unlock = function (operation) {
-    assert.strictEqual(typeof operation, 'string');
-
-    if (this._operation !== operation) throw new BoxError(BoxError.BAD_STATE, 'Mismatched unlock. Current lock is for ' + this._operation); // throw because this is a programming error
-
-    if (--this._lockDepth === 0) {
-        debug('Released : %s', this._operation);
-
-        this._operation = null;
-        this._timestamp = null;
-        clearInterval(this._watcherId);
-        this._watcherId = -1;
-    } else {
-        debug('Recursive lock released : %s. Depth : %s', this._operation, this._lockDepth);
-    }
-
-    this.emit('unlocked', operation);
-
-    return null;
-};
-
-exports = module.exports = new Locker();

src/locks.js

@@ -0,0 +1,121 @@
+'use strict';
+
+exports = module.exports = {
+    setTaskId,
+
+    acquire,
+    wait,
+
+    release,
+    releaseAll,
+    releaseByTaskId,
+
+    TYPE_APP_PREFIX: 'app_',
+    TYPE_UPDATE: 'update',
+    TYPE_UPDATE_TASK: 'update_task',
+    TYPE_BACKUP_TASK: 'backup_task',
+
+    TYPE_MAIL_SERVER_RESTART: 'mail_restart',
+};
+
+const assert = require('assert'),
+    BoxError = require('./boxerror.js'),
+    database = require('./database.js'),
+    debug = require('debug')('box:locks'),
+    promiseRetry = require('./promise-retry.js');
+
+let gTaskId = null;
+
+function setTaskId(taskId) {
+    assert.strictEqual(typeof taskId, 'string');
+    gTaskId = taskId;
+}
+
+async function read() {
+    const result = await database.query('SELECT version, dataJson FROM locks');
+    return { version: result[0].version, data: JSON.parse(result[0].dataJson) };
+}
+
+async function write(value) {
+    assert.strictEqual(typeof value.version, 'number');
+    assert.strictEqual(typeof value.data, 'object');
+
+    const result = await database.query('UPDATE locks SET dataJson=?, version=version+1 WHERE id=? AND version=?', [ JSON.stringify(value.data), 'platform', value.version ]);
+    if (result.affectedRows !== 1) throw new BoxError(BoxError.CONFLICT, 'Someone updated before we did');
+    debug(`write: current locks: ${JSON.stringify(value.data)}`);
+}
+
+function canAcquire(data, type) {
+    assert.strictEqual(typeof data, 'object');
+    assert.strictEqual(typeof type, 'string');
+
+    if (type === exports.TYPE_UPDATE) {
+        if (Object.keys(data).some(k => k.startsWith('app-'))) return new BoxError(BoxError.BAD_STATE, 'One or more apptasks are active');
+    } else if (type.startsWith(exports.TYPE_APP_PREFIX)) {
+        if (exports.TYPE_UPDATE in data) return new BoxError(BoxError.BAD_STATE, 'Update is active');
+    } else if (type === exports.TYPE_BACKUP_TASK) {
+        if (exports.TYPE_UPDATE_TASK in data) return new BoxError(BoxError.BAD_STATE, 'Update task is active');
+    } else if (type === exports.TYPE_UPDATE_TASK) {
+        if (exports.TYPE_BACKUP_TASK in data) return new BoxError(BoxError.BAD_STATE, 'Backup task is active');
+    }
+
+    return null;
+}
+
+async function acquire(type) {
+    assert.strictEqual(typeof type, 'string');
+
+    await promiseRetry({ times: Number.MAX_SAFE_INTEGER, interval: 100, debug, retry: (error) => error.reason === BoxError.CONFLICT }, async () => {
+        const { version, data } = await read();
+        if (type in data) throw new BoxError(BoxError.BAD_STATE, `Locked by ${data[type]}`);
+        const error = canAcquire(data, type);
+        if (error) throw error;
+        data[type] = gTaskId;
+        await write({ version, data });
+        debug(`acquire: ${type}`);
+    });
+}
+
+async function wait(type) {
+    assert.strictEqual(typeof type, 'string');
+
+    await promiseRetry({ times: Number.MAX_SAFE_INTEGER, interval: 10000, debug }, async () => await acquire(type));
+}
+
+async function release(type) {
+    assert.strictEqual(typeof type, 'string');
+
+    await promiseRetry({ times: Number.MAX_SAFE_INTEGER, interval: 100, debug, retry: (error) => error.reason === BoxError.CONFLICT }, async () => {
+        const { version, data } = await read();
+        if (!(type in data)) throw new BoxError(BoxError.BAD_STATE, `Lock ${type} was never acquired`);
+        if (data[type] !== gTaskId) throw new BoxError(BoxError.BAD_STATE, `Task ${gTaskId} attempted to release lock ${type} acquired by ${data[type]}`);
+        delete data[type];
+        await write({ version, data });
+        debug(`release: ${type}`);
+    });
+}
+
+async function releaseAll() {
+    await database.query('DELETE FROM locks');
+    await database.query('INSERT INTO locks (id, dataJson) VALUES (?, ?)', [ 'platform', JSON.stringify({}) ]);
+    debug('releaseAll: all locks released');
+}
+
+async function releaseByTaskId(taskId) {
+    assert.strictEqual(typeof taskId, 'string');
+
+    await promiseRetry({ times: Number.MAX_SAFE_INTEGER, interval: 100, debug, retry: (error) => error.reason === BoxError.CONFLICT }, async () => {
+        const { version, data } = await read();
+
+        for (const type of Object.keys(data)) {
+            if (data[type] === taskId) {
+                debug(`releaseByTaskId: task ${taskId} forgot to unlock ${type}`);
+                delete data[type];
+            }
+        }
+
+        await write({ version, data });
+
+        debug(`releaseByTaskId: ${taskId}`);
+    });
+}

src/mail.js

@@ -684,7 +684,7 @@ async function upsertDnsRecords(domain, mailFqdn) {
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof mailFqdn, 'string');
 
-    debug(`upsertDnsRecords: updating mail dns records of domain ${domain} and mail fqdn ${mailFqdn}`);
+    debug(`upsertDnsRecords: updating mail dns records domain:${domain} mailFqdn:${mailFqdn}`);
 
     const mailDomain = await getDomain(domain);
     if (!mailDomain) throw new BoxError(BoxError.NOT_FOUND, 'mail domain not found');
@@ -706,7 +706,7 @@ async function upsertDnsRecords(domain, mailFqdn) {
     const dmarcRecords = await dns.getDnsRecords('_dmarc', domain, 'TXT'); // only update dmarc if absent. this allows user to set email for reporting
     if (dmarcRecords.length === 0) records.push({ subdomain: '_dmarc', domain, type: 'TXT', values: [ '"v=DMARC1; p=reject; pct=100"' ] });
 
-    debug(`upsertDnsRecords: will update ${domain} with ${JSON.stringify(records)}`);
+    debug(`upsertDnsRecords: updating ${domain} with ${records.length} records: ${JSON.stringify(records)}`);
 
     for (const record of records) {
         await dns.upsertDnsRecords(record.subdomain, record.domain, record.type, record.values);
@@ -1210,8 +1210,8 @@ async function resolveList(listName, listDomain) {
 async function checkStatus() {
     const result = await checkConfiguration();
     if (result.status) {
-        await notifications.clearAlert(notifications.ALERT_MAIL_STATUS, 'Email is not configured properly');
+        await notifications.unpin(notifications.TYPE_MAIL_STATUS, {});
     } else {
-        await notifications.alert(notifications.ALERT_MAIL_STATUS, 'Email is not configured properly', result.message, { persist: true });
+        await notifications.pin(notifications.TYPE_MAIL_STATUS, 'Email is not configured properly', result.message, {});
     }
 }

src/mail_templates/app_down-text.ejs

@@ -0,0 +1,19 @@
+Dear Cloudron Admin,
+
+The application '<%= title %>' installed at <%= appFqdn %> is not responding.
+
+This is most likely a problem in the application.
+
+To resolve this, you can try the following:
+
+* Check the app logs - https://docs.cloudron.io/apps/#log-viewer
+* Restart the app from the Recovery section of the app - https://docs.cloudron.io/apps/#restart-app
+* Check the troubleshooting guidelines - https://docs.cloudron.io/troubleshooting/#unresponsive-app
+* Contact us in our Forum at https://forum.cloudron.io
+
+Powered by https://cloudron.io
+
+Don't want such mails? Change your notification preferences at <%= notificationsUrl %>
+
+Sent at: <%= new Date().toUTCString() %>
+

src/mail_templates/app_up-text.ejs

@@ -0,0 +1,11 @@
+Dear Cloudron Admin,
+
+The application '<%= title %>' installed at <%= appFqdn %> is back online
+and responding to health checks.
+
+Powered by https://cloudron.io
+
+Don't want such mails? Change your notification preferences at <%= notificationsUrl %>
+
+Sent at: <%= new Date().toUTCString() %>
+

src/mail_templates/backup_failed-text.ejs

@@ -1,8 +1,6 @@
-<%if (format === 'text') { %>
-
 Dear <%= cloudronName %> Admin,
 
-Cloudron failed to create a complete backup. Please see the logs at <%= logUrl %> for more information.
+Cloudron failed to create a backup. Please see the logs at <%= logUrl %> for more information.
 
 -------------------------------------
 
@@ -13,8 +11,7 @@ Cloudron failed to create a complete backup. Please see the logs at <%= logUrl %
 
 Powered by https://cloudron.io
 
+Don't want such mails? Change your notification preferences at <%= notificationsUrl %>
+
 Sent at: <%= new Date().toUTCString() %>
 
-<% } else { %>
-
-<% } %>

src/mail_templates/certificate_renewal_error-text.ejs

@@ -1,5 +1,3 @@
-<%if (format === 'text') { %>
-
 Dear Cloudron Admin,
 
 The certificate for <%= domain %> could not be renewed.
@@ -23,8 +21,7 @@ The error was:
 
 Powered by https://cloudron.io
 
+Don't want such mails? Change your notification preferences at <%= notificationsUrl %>
+
 Sent at: <%= new Date().toUTCString() %>
 
-<% } else { %>
-
-<% } %>

src/mail_templates/oom_event-text.ejs

@@ -0,0 +1,24 @@
+Dear <%= cloudronName %> Admin,
+
+<%if (app) { %>
+The application at <%= app.fqdn %> ran out of memory. The application has been restarted automatically. If you see this notification often,
+consider increasing the memory limit - <%= webadminUrl %>/#/app/<%= app.id %>/resources .
+<% } else { %>
+The addon <%= addon.name %> service ran out of memory. The service has been restarted automatically. If you see this notification often,
+consider increasing the memory limit - <%= webadminUrl %>/#/services .
+<% } %>
+
+Out of memory event:
+
+-------------------------------------
+
+<%- event %>
+
+-------------------------------------
+
+Powered by https://cloudron.io
+
+Don't want such mails? Change your notification preferences at <%= notificationsUrl %>
+
+Sent at: <%= new Date().toUTCString() %>
+