Fix crash when only udp ports are defined

This reverts commit a19205e3ad.

CHANGES

@@ -1674,3 +1674,28 @@
 * Fix issue where sieve responses were not sent via the relay
 * File based session store
 * Fix API token error reporting for namecheap backend
+
+[4.2.2]
+* Fix typos in migration
+
+[4.2.3]
+* Remove flicker of custom icon
+* Preserve PROVIDER setting from cloudron.conf
+* Add Skip backup option when updating an app
+* Fix bug where nginx was not reloaded on cert renewal
+
+[4.2.4]
+* Fix demo settings state regression
+
+[4.2.5]
+* Fix the demo settins fix
+
+[4.2.6]
+* Fix configuration of empty app location (subdomain)
+
+[4.2.7]
+* Fix issue where the icon for normal users was displayed incorrectly
+* Kill stuck backup processes after 12 hours and notify admins
+* Reconfigure email apps when mail domain is added/removed
+* Fix crash when only udp ports are defined
+

migrations/20190725172940-settings-migrate-cloudron-conf.js

@@ -1,8 +1,7 @@
 'use strict';
 
 var async = require('async'),
-    fs = require('fs'),
-    superagent = require('superagent');
+    fs = require('fs');
 
 exports.up = function(db, callback) {
     if (!fs.existsSync('/etc/cloudron/cloudron.conf')) {
@@ -13,6 +12,7 @@ exports.up = function(db, callback) {
     const config = JSON.parse(fs.readFileSync('/etc/cloudron/cloudron.conf', 'utf8'));
 
     async.series([
+        fs.writeFile.bind(null, '/etc/cloudron/PROVIDER', config.provider, 'utf8'),
         db.runSql.bind(db, 'START TRANSACTION;'),
         // we use replace instead of insert because the cloudron-setup adds api/web_server_origin even for legacy setups
         db.runSql.bind(db, 'REPLACE INTO settings (name, value) VALUES(?, ?)', [ 'api_server_origin', config.apiServerOrigin ]),

migrations/20191002103646-fix-demo-settings-boolean.js

@@ -0,0 +1,10 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    // We clear all demo state in the Cloudron...the demo cloudron needs manual intervention afterwards
+    db.runSql('REPLACE INTO settings (name, value) VALUES(?, ?)', [ 'demo', '' ], callback);
+};
+
+exports.down = function(db, callback) {
+    callback();
+};

src/apps.js

@@ -365,12 +365,11 @@ function validateDataDir(dataDir) {
     return null;
 }
 
-function getDuplicateErrorDetails(errorMessage, location, domainObject, portBindings, alternateDomains) {
+function getDuplicateErrorDetails(errorMessage, locations, domainObjectMap, portBindings) {
     assert.strictEqual(typeof errorMessage, 'string');
-    assert.strictEqual(typeof location, 'string');
-    assert.strictEqual(typeof domainObject, 'object');
+    assert(Array.isArray(locations));
+    assert.strictEqual(typeof domainObjectMap, 'object');
     assert.strictEqual(typeof portBindings, 'object');
-    assert(Array.isArray(alternateDomains));
 
     var match = errorMessage.match(/ER_DUP_ENTRY: Duplicate entry '(.*)' for key '(.*)'/);
     if (!match) {
@@ -378,17 +377,13 @@ function getDuplicateErrorDetails(errorMessage, location, domainObject, portBind
         return new AppsError(AppsError.INTERNAL_ERROR, new Error(errorMessage));
     }
 
-    // check if the location or alternateDomains conflicts
+    // check if a location conflicts
     if (match[2] === 'subdomain') {
-        // mysql reports a unique conflict with a dash: eg. domain:example.com subdomain:test => test-example.com
-        if (match[1] === `${location}-${domainObject.domain}`) {
-            return new AppsError(AppsError.ALREADY_EXISTS, `Domain '${domains.fqdn(location, domainObject)}' is in use`, { location: location, domain: domainObject.domain });
-        }
+        for (let i = 0; i < locations.length; i++) {
+            const { subdomain, domain } = locations[i];
+            if  (match[1] !== `${subdomain}-${domain}`) continue;
 
-        for (let d of alternateDomains) {
-            if  (match[1] !== `${d.subdomain}-${d.domain}`) continue;
-
-            return new AppsError(AppsError.ALREADY_EXISTS, `Alternate domain '${d.subdomain}.${d.domain}' is in use`, { location: d.subdomain, domain: d.domain });
+            return new AppsError(AppsError.ALREADY_EXISTS, `Domain '${domains.fqdn(subdomain, domainObjectMap[domain])}' is in use`, { subdomain, domain });
         }
     }
 
@@ -686,6 +681,24 @@ function checkAppState(app, state) {
     return null;
 }
 
+function validateLocations(locations, callback) {
+    assert(Array.isArray(locations));
+    assert.strictEqual(typeof callback, 'function');
+
+    getDomainObjectMap(function (error, domainObjectMap) {
+        if (error) return callback(error);
+
+        for (let location of locations) {
+            if (!(location.domain in domainObjectMap)) return callback(new AppsError(AppsError.BAD_FIELD, 'No such domain', { field: 'location', domain: location.domain, subdomain: location.subdomain }));
+
+            error = domains.validateHostname(location.subdomain, domainObjectMap[location.domain]);
+            if (error) return callback(new AppsError(AppsError.BAD_FIELD, 'Bad location: ' + error.message, { field: 'location', domain: location.domain, subdomain: location.subdomain }));
+        }
+
+        callback(null, domainObjectMap);
+    });
+}
+
 function install(data, user, auditSource, callback) {
     assert(data && typeof data === 'object');
     assert(user && typeof user === 'object');
@@ -773,15 +786,12 @@ function install(data, user, auditSource, callback) {
             }
         }
 
-        domains.get(domain, function (error, domainObject) {
-            if (error && error.reason === DomainsError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND, 'No such domain'));
-            if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, 'Could not get domain info:' + error.message));
-
-            error = domains.validateHostname(location, domainObject);
-            if (error) return callback(new AppsError(AppsError.BAD_FIELD, 'Bad location: ' + error.message, { field: 'location' }));
+        const locations = [{subdomain: location, domain}].concat(alternateDomains);
+        validateLocations(locations, function (error, domainObjectMap) {
+            if (error) return callback(error);
 
             if (cert && key) {
-                error = reverseProxy.validateCertificate(location, domainObject, { cert, key });
+                error = reverseProxy.validateCertificate(location, domainObjectMap[domain], { cert, key });
                 if (error) return callback(new AppsError(AppsError.BAD_FIELD, error.message, { field: 'cert' }));
             }
 
@@ -801,11 +811,11 @@ function install(data, user, auditSource, callback) {
                 label: label,
                 tags: tags,
                 runState: exports.RSTATE_RUNNING,
-                installationState: exports.ISTATE_PENDING_INSTALL
+                installationState: backupId ? exports.ISTATE_PENDING_RESTORE : exports.ISTATE_PENDING_INSTALL
             };
 
             appdb.add(appId, appStoreId, manifest, location, domain, translatePortBindings(portBindings, manifest), data, function (error) {
-                if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(getDuplicateErrorDetails(error.message, location, domainObject, portBindings, data.alternateDomains));
+                if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(getDuplicateErrorDetails(error.message, locations, domainObjectMap, portBindings));
                 if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND, error.message));
                 if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
@@ -814,7 +824,7 @@ function install(data, user, auditSource, callback) {
 
                     // save cert to boxdata/certs
                     if (cert && key) {
-                        let error = reverseProxy.setAppCertificateSync(location, domainObject, { cert, key });
+                        let error = reverseProxy.setAppCertificateSync(location, domainObjectMap[domain], { cert, key });
                         if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, 'Error setting cert: ' + error.message));
                     }
 
@@ -822,13 +832,16 @@ function install(data, user, auditSource, callback) {
                     const task = {
                         args: { restoreConfig, overwriteDns },
                         values: { },
-                        requiredState: exports.ISTATE_PENDING_INSTALL
+                        requiredState: data.installationState
                     };
 
-                    addTask(appId, exports.ISTATE_PENDING_INSTALL, task, function (error, result) {
+                    addTask(appId, data.installationState, task, function (error, result) {
                         if (error) return callback(error);
 
-                        const newApp = _.extend({ fqdn: domains.fqdn(location, domainObject) }, data, { appStoreId, manifest, location, domain, portBindings });
+                        const newApp = _.extend({}, data, { appStoreId, manifest, location, domain, portBindings });
+                        newApp.fqdn = domains.fqdn(newApp.location, domainObjectMap[newApp.domain]);
+                        newApp.alternateDomains.forEach(function (ad) { ad.fqdn = domains.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });
+
                         eventlog.add(eventlog.ACTION_APP_INSTALL, auditSource, { appId: appId, app: newApp, taskId: result.taskId });
 
                         callback(null, { id : appId, taskId: result.taskId });
@@ -1162,14 +1175,13 @@ function setLocation(appId, data, auditSource, callback) {
         if (error) return callback(error);
 
         let values = {
+            location: data.location.toLowerCase(),
+            domain: data.domain.toLowerCase(),
             // these are intentionally reset, if not set
             portBindings: null,
             alternateDomains: []
         };
 
-        values.location = data.location.toLowerCase();
-        values.domain = data.domain.toLowerCase();
-
         if ('portBindings' in data) {
             error = validatePortBindings(data.portBindings, app.manifest);
             if (error) return callback(error);
@@ -1177,20 +1189,17 @@ function setLocation(appId, data, auditSource, callback) {
             values.portBindings = translatePortBindings(data.portBindings || null, app.manifest);
         }
 
-        if ('alternateDomains' in data) {
-            // TODO validate all subdomains [{ domain: '', subdomain: ''}]
-            values.alternateDomains = data.alternateDomains;
-        }
-
         // move the mailbox name to match the new location
         if (app.mailboxName.endsWith('.app')) values.mailboxName = mailboxNameForLocation(values.location, app.manifest);
 
-        domains.get(values.domain, function (error, domainObject) {
-            if (error && error.reason === DomainsError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND, 'No such domain'));
-            if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, 'Could not get domain info:' + error.message));
+        if ('alternateDomains' in data) {
+            values.alternateDomains = data.alternateDomains;
+        }
 
-            error = domains.validateHostname(values.location, domainObject);
-            if (error) return callback(new AppsError(AppsError.BAD_FIELD, 'Bad location: ' + error.message, { field: 'location' }));
+        const locations = [{subdomain: values.location, domain: values.domain}].concat(values.alternateDomains);
+
+        validateLocations(locations, function (error, domainObjectMap) {
+            if (error) return callback(error);
 
             const task = {
                 args: {
@@ -1200,10 +1209,13 @@ function setLocation(appId, data, auditSource, callback) {
                 values
             };
             addTask(appId, exports.ISTATE_PENDING_LOCATION_CHANGE, task, function (error, result) {
-                if (error && error.reason === AppsError.ALREADY_EXISTS) error = getDuplicateErrorDetails(error.message, values.location, domainObject, data.portBindings, app.alternateDomains);
+                if (error && error.reason === AppsError.ALREADY_EXISTS) error = getDuplicateErrorDetails(error.message, locations, domainObjectMap, data.portBindings);
                 if (error) return callback(error);
 
-                eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId: appId, app: app, location: values.location, domain: values.domain, portBindings: values.portBindings, alternateDomains: values.alternateDomains, taskId: result.taskId });
+                values.fqdn = domains.fqdn(values.location, domainObjectMap[values.domain]);
+                values.alternateDomains.forEach(function (ad) { ad.fqdn = domains.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });
+
+                eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, _.extend({ appId, app, taskId: result.taskId }, values));
 
                 callback(null, { taskId: result.taskId });
             });
@@ -1248,6 +1260,8 @@ function update(appId, data, auditSource, callback) {
 
     debug(`update: id:${appId}`);
 
+    const skipBackup = !!data.skipBackup;
+
     get(appId, function (error, app) {
         if (error) return callback(error);
 
@@ -1257,18 +1271,13 @@ function update(appId, data, auditSource, callback) {
         downloadManifest(data.appStoreId, data.manifest, function (error, appStoreId, manifest) {
             if (error) return callback(error);
 
-            var updateConfig = {
-                skipNotification: data.force,
-                skipBackup: data.force
-            };
-
             error = manifestFormat.parse(manifest);
             if (error) return callback(new AppsError(AppsError.BAD_FIELD, 'Manifest error:' + error.message));
 
             error = checkManifestConstraints(manifest);
             if (error) return callback(error);
 
-            updateConfig.manifest = manifest;
+            var updateConfig = { skipBackup, manifest };
 
             // prevent user from installing a app with different manifest id over an existing app
             // this allows cloudron install -f --app <appid> for an app installed from the appStore
@@ -1313,7 +1322,7 @@ function update(appId, data, auditSource, callback) {
             addTask(appId, exports.ISTATE_PENDING_UPDATE, task, function (error, result) {
                 if (error) return callback(error);
 
-                eventlog.add(eventlog.ACTION_APP_UPDATE, auditSource, { appId: appId, toManifest: manifest, fromManifest: app.manifest, force: data.force, app: app, taskId: result.taskId });
+                eventlog.add(eventlog.ACTION_APP_UPDATE, auditSource, { appId, app, skipBackup, toManifest: manifest, fromManifest: app.manifest, force: data.force, taskId: result.taskId });
 
                 // clear update indicator, if update fails, it will come back through the update checker
                 updateChecker.resetAppUpdateInfo(appId);
@@ -1388,26 +1397,35 @@ function repair(appId, data, auditSource, callback) {
 
     get(appId, function (error, app) {
         if (error) return callback(error);
-        if (app.installationState !== exports.ISTATE_ERROR) return new AppsError(AppsError.BAD_STATE, 'Only allowed in error state');
 
-        const appError = app.error || {};
+        const appError = app.error || {}; // repair can always be called
         const newState = appError.installationState ? appError.installationState : exports.ISTATE_PENDING_CONFIGURE;
+
         debug(`Repairing app with error: ${JSON.stringify(error)} and state: ${newState}`);
 
-        let values = _.pick(data, 'location', 'domain', 'alternateDomains'); // FIXME: validate
+        let values = _.pick(data, 'location', 'domain', 'alternateDomains');
 
-        tasks.get(appError.taskId || '', function (error, task) {
-            let args = !error ? task.args[1] : {}; // the first argument is the app id
-            args.restoreConfig = data.backupId ? { backupId: data.backupId, backupFormat: data.backupFormat, oldManifest: app.manifest } : null; // when null, apptask simply reinstalls
-            args.overwriteDns = 'overwriteDns' in data ? data.overwriteDns : false;
+        const locations = (values.location ? [ { subdomain: values.location, domain: values.domain } ] : []).concat(values.alternateDomains || []);
+        validateLocations(locations, function (error, domainObjectMap) {
+            if (error) return callback(error);
 
-            // create a new task instead of updating the old one, since it helps tracking
-            addTask(appId, newState, { args, values, requiredState: exports.ISTATE_ERROR }, function (error, result) {
-                if (error) return callback(error);
+            tasks.get(appError.taskId || '', function (error, task) {
+                let args = !error ? task.args[1] : {}; // pick args for the failed task. the first argument is the app id
 
-                eventlog.add(eventlog.ACTION_APP_REPAIR, auditSource, { taskId: result.taskId, app, newState });
+                if ('backupId' in data) {
+                    args.restoreConfig = data.backupId ? { backupId: data.backupId, backupFormat: data.backupFormat, oldManifest: app.manifest } : null; // when null, apptask simply reinstalls
+                }
+                args.overwriteDns = 'overwriteDns' in data ? data.overwriteDns : false;
 
-                callback(null, { taskId: result.taskId });
+                // create a new task instead of updating the old one, since it helps tracking
+                addTask(appId, newState, { args, values, requiredState: null }, function (error, result) {
+                    if (error && error.reason === AppsError.ALREADY_EXISTS) error = getDuplicateErrorDetails(error.message, locations, domainObjectMap, { /* portBindings */});
+                    if (error) return callback(error);
+
+                    eventlog.add(eventlog.ACTION_APP_REPAIR, auditSource, { taskId: result.taskId, app, newState });
+
+                    callback(null, { taskId: result.taskId });
+                });
             });
         });
     });
@@ -1441,7 +1459,7 @@ function restore(appId, data, auditSource, callback) {
             error = checkManifestConstraints(backupInfo.manifest);
             if (error) return callback(error);
 
-            const restoreConfig = data.backupId ? { backupId: data.backupId, backupFormat: backupInfo.format, oldManifest: app.manifest } : null; // when null, apptask simply reinstalls
+            const restoreConfig = { backupId: data.backupId, backupFormat: backupInfo.format, oldManifest: app.manifest };
 
             const task = {
                 args: {
@@ -1533,12 +1551,9 @@ function clone(appId, data, user, auditSource, callback) {
                 mailboxName = mailboxNameForLocation(location, manifest);
             }
 
-            domains.get(domain, function (error, domainObject) {
-                if (error && error.reason === DomainsError.NOT_FOUND) return callback(new AppsError(AppsError.EXTERNAL_ERROR, 'No such domain'));
-                if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, 'Could not get domain info:' + error.message));
-
-                error = domains.validateHostname(location, domainObject);
-                if (error) return callback(new AppsError(AppsError.BAD_FIELD, 'Bad location: ' + error.message));
+            const locations = [{subdomain: location, domain}];
+            validateLocations(locations, function (error, domainObjectMap) {
+                if (error) return callback(error);
 
                 var newAppId = uuid.v4();
 
@@ -1551,17 +1566,18 @@ function clone(appId, data, user, auditSource, callback) {
                     mailboxName: mailboxName,
                     enableBackup: app.enableBackup,
                     robotsTxt: app.robotsTxt,
-                    env: app.env
+                    env: app.env,
+                    alternateDomains: []
                 };
 
                 appdb.add(newAppId, appStoreId, manifest, location, domain, translatePortBindings(portBindings, manifest), data, function (error) {
-                    if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(getDuplicateErrorDetails(error.message, location, domainObject, portBindings, []));
+                    if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(getDuplicateErrorDetails(error.message, locations, domainObjectMap, portBindings));
                     if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
                     purchaseApp({ appId: newAppId, appstoreId: app.appStoreId, manifestId: manifest.id }, function (error) {
                         if (error) return callback(error);
 
-                        const restoreConfig = { backupId: backupId, backupFormat: backupInfo.format };
+                        const restoreConfig = { backupId: backupId, backupFormat: backupInfo.format, oldManifest: null };
                         const task = {
                             args: { restoreConfig, overwriteDns },
                             values: {},
@@ -1571,6 +1587,8 @@ function clone(appId, data, user, auditSource, callback) {
                             if (error) return callback(error);
 
                             const newApp = _.extend({}, data, { appStoreId, manifest, location, domain, portBindings });
+                            newApp.fqdn = domains.fqdn(newApp.location, domainObjectMap[newApp.domain]);
+                            newApp.alternateDomains.forEach(function (ad) { ad.fqdn = domains.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });
                             eventlog.add(eventlog.ACTION_APP_CLONE, auditSource, { appId: newAppId, oldAppId: appId, backupId: backupId, oldApp: app, newApp: newApp, taskId: result.taskId });
 
                             callback(null, { id: newAppId, taskId: result.taskId });
@@ -1849,7 +1867,15 @@ function restoreInstalledApps(callback) {
 
         async.eachSeries(apps, function (app, iteratorDone) {
             backups.getByAppIdPaged(1, 1, app.id, function (error, results) {
-                const restoreConfig = !error && results.length ? { backupId: results[0].id, backupFormat: results[0].format, oldManifest: app.manifest } : null;
+                let installationState, restoreConfig;
+                if (!error && results.length) {
+                    installationState = exports.ISTATE_PENDING_RESTORE;
+                    restoreConfig = { backupId: results[0].id, backupFormat: results[0].format, oldManifest: app.manifest };
+                } else {
+                    installationState = exports.ISTATE_PENDING_INSTALL;
+                    restoreConfig = null;
+                }
+
                 const task = {
                     args: { restoreConfig, overwriteDns: true },
                     values: {},
@@ -1859,7 +1885,7 @@ function restoreInstalledApps(callback) {
 
                 debug(`restoreInstalledApps: marking ${app.fqdn} for restore using restore config ${JSON.stringify(restoreConfig)}`);
 
-                addTask(app.id, exports.ISTATE_PENDING_RESTORE, task, function (error, result) {
+                addTask(app.id, installationState, task, function (error, result) {
                     if (error) debug(`restoreInstalledApps: error marking ${app.fqdn} for restore: ${JSON.stringify(error)}`);
                     else debug(`restoreInstalledApps: marked ${app.id} for restore with taskId ${result.taskId}`);
 
@@ -1883,7 +1909,7 @@ function configureInstalledApps(callback) {
 
             const oldConfig = _.pick(app, 'location', 'domain', 'fqdn', 'alternateDomains', 'portBindings');
             const task = {
-                args: { oldConfig },
+                args: { oldConfig, overwriteDns: true },
                 values: {},
                 scheduleNow: false, // task will be scheduled by autoRestartTasks when platform is ready
                 requireNullTaskId: false // ignore existing stale taskId

src/apptask.js

@@ -511,8 +511,7 @@ function install(app, args, progressCallback, callback) {
     assert.strictEqual(typeof progressCallback, 'function');
     assert.strictEqual(typeof callback, 'function');
 
-    const isInstalling = app.installationState !== apps.ISTATE_PENDING_RESTORE; // install or clone or repair
-    const restoreConfig = args.restoreConfig || {};
+    const restoreConfig = args.restoreConfig; // has to be set when restoring
     const overwriteDns = args.overwriteDns;
 
     async.series([
@@ -529,15 +528,19 @@ function install(app, args, progressCallback, callback) {
         deleteContainers.bind(null, app, { managedOnly: true }),
         function teardownAddons(next) {
             // when restoring, app does not require these addons anymore. remove carefully to preserve the db passwords
-            var addonsToRemove = isInstalling ? app.manifest.addons : _.omit(restoreConfig.oldManifest.addons, Object.keys(app.manifest.addons));
+            let addonsToRemove;
+            if (restoreConfig && restoreConfig.oldManifest) { // oldManifest is null for clone
+                addonsToRemove = _.omit(restoreConfig.oldManifest.addons, Object.keys(app.manifest.addons));
+            } else {
+                addonsToRemove = app.manifest.addons;
+            }
 
             addons.teardownAddons(app, addonsToRemove, next);
         },
         deleteAppDir.bind(null, app, { removeDirectory: false }), // do not remove any symlinked appdata dir
 
-        // for restore case
         function deleteImageIfChanged(done) {
-            if (isInstalling) return done();
+            if (!restoreConfig || !restoreConfig.oldManifest) return done();
 
             if (restoreConfig.oldManifest.dockerImage === app.manifest.dockerImage) return done();
 
@@ -559,7 +562,7 @@ function install(app, args, progressCallback, callback) {
         createAppDir.bind(null, app),
 
         function restoreFromBackup(next) {
-            if (!restoreConfig.backupId) {
+            if (!restoreConfig) {
                 async.series([
                     progressCallback.bind(null, { percent: 60, message: 'Setting up addons' }),
                     addons.setupAddons.bind(null, app, app.manifest.addons),
@@ -667,6 +670,7 @@ function changeLocation(app, args, progressCallback, callback) {
 
     const oldConfig = args.oldConfig;
     const locationChanged = oldConfig.fqdn !== app.fqdn;
+    const overwriteDns = args.overwriteDns;
 
     async.series([
         progressCallback.bind(null, { percent: 10, message: 'Cleaning up old install' }),
@@ -686,7 +690,7 @@ function changeLocation(app, args, progressCallback, callback) {
         },
 
         progressCallback.bind(null, { percent: 30, message: 'Registering subdomains' }),
-        registerSubdomains.bind(null, app, args.overwriteDns),
+        registerSubdomains.bind(null, app, overwriteDns),
 
         // re-setup addons since they rely on the app's fqdn (e.g oauth)
         progressCallback.bind(null, { percent: 50, message: 'Setting up addons' }),
@@ -767,6 +771,9 @@ function configure(app, args, progressCallback, callback) {
     assert.strictEqual(typeof progressCallback, 'function');
     assert.strictEqual(typeof callback, 'function');
 
+    const oldConfig = args.oldConfig || null;
+    const overwriteDns = args.overwriteDns;
+
     async.series([
         progressCallback.bind(null, { percent: 10, message: 'Cleaning up old install' }),
         unconfigureReverseProxy.bind(null, app),
@@ -775,13 +782,13 @@ function configure(app, args, progressCallback, callback) {
         docker.stopContainers.bind(null, app.id),
         deleteContainers.bind(null, app, { managedOnly: true }),
         function (next) {
-            if (!args.oldConfig) return next();
+            if (!oldConfig) return next();
 
-            let obsoleteDomains = args.oldConfig.alternateDomains.filter(function (o) {
+            let obsoleteDomains = oldConfig.alternateDomains.filter(function (o) {
                 return !app.alternateDomains.some(function (n) { return n.subdomain === o.subdomain && n.domain === o.domain; });
             });
 
-            if (args.oldConfig.fqdn !== app.fqdn) obsoleteDomains.push({ subdomain: args.oldConfig.location, domain: args.oldConfig.domain });
+            if (oldConfig.fqdn !== app.fqdn) obsoleteDomains.push({ subdomain: oldConfig.location, domain: oldConfig.domain });
 
             if (obsoleteDomains.length === 0) return next();
 
@@ -794,7 +801,7 @@ function configure(app, args, progressCallback, callback) {
         downloadIcon.bind(null, app),
 
         progressCallback.bind(null, { percent: 30, message: 'Registering subdomains' }),
-        registerSubdomains.bind(null, app, false /* overwrite */), // if location changed, do not overwrite to detect conflicts
+        registerSubdomains.bind(null, app, overwriteDns),
 
         progressCallback.bind(null, { percent: 40, message: 'Downloading image' }),
         downloadImage.bind(null, app.manifest),
@@ -932,9 +939,7 @@ function update(app, args, progressCallback, callback) {
             debugApp(app, 'Error updating app: %s', error);
             updateApp(app, { installationState: apps.ISTATE_ERROR, error: makeTaskError(error, app) }, callback.bind(null, error));
         } else {
-            if (updateConfig.skipNotification) return callback(null);
-
-            eventlog.add(eventlog.ACTION_APP_UPDATE_FINISH, auditsource.APP_TASK, { app: app, success: true }, callback);
+            eventlog.add(eventlog.ACTION_APP_UPDATE_FINISH, auditsource.APP_TASK, { app: app, success: true }, () => callback()); // ignore error
         }
     });
 }

src/backups.js

@@ -1024,12 +1024,13 @@ function startBackupTask(auditSource, callback) {
 
         eventlog.add(eventlog.ACTION_BACKUP_START, auditSource, { taskId });
 
-        tasks.startTask(taskId, {}, function (error, result) {
+        tasks.startTask(taskId, { timeout: 12 * 60 * 60 * 1000 /* 12 hours */ }, function (error, backupId) {
             locker.unlock(locker.OP_FULL_BACKUP);
 
             const errorMessage = error ? error.message : '';
+            const timedOut = error ? error.code === tasks.ETIMEOUT : false;
 
-            eventlog.add(eventlog.ACTION_BACKUP_FINISH, auditSource, { taskId: taskId, errorMessage: errorMessage, backupId: result });
+            eventlog.add(eventlog.ACTION_BACKUP_FINISH, auditSource, { taskId, errorMessage, timedOut, backupId });
         });
 
         callback(null, taskId);
@@ -1261,10 +1262,9 @@ function startCleanupTask(auditSource, callback) {
     tasks.add(tasks.TASK_CLEAN_BACKUPS, [ auditSource ], function (error, taskId) {
         if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
 
-        eventlog.add(eventlog.ACTION_BACKUP_CLEANUP_START, auditSource, { taskId });
-
         tasks.startTask(taskId, {}, (error, result) => { // result is { removedBoxBackups, removedAppBackups }
             eventlog.add(eventlog.ACTION_BACKUP_CLEANUP_FINISH, auditSource, {
+                taskId,
                 errorMessage: error ? error.message : null,
                 removedBoxBackups: result ? result.removedBoxBackups : [],
                 removedAppBackups: result ? result.removedAppBackups : []

src/boxerror.js

@@ -45,8 +45,10 @@ BoxError.INTERNAL_ERROR = 'Internal Error';
 BoxError.LOGROTATE_ERROR = 'Logrotate Error';
 BoxError.NETWORK_ERROR = 'Network Error';
 BoxError.NOT_FOUND = 'Not found';
+BoxError.OPENSSL_ERROR = 'OpenSSL Error';
 BoxError.REVERSEPROXY_ERROR = 'ReverseProxy Error';
 BoxError.TASK_ERROR = 'Task Error';
+BoxError.TRY_AGAIN = 'Try Again';
 BoxError.UNKNOWN_ERROR = 'Unknown Error'; // only used for porting
 
 BoxError.prototype.toPlainObject = function () {

src/cert/acme2.js

@@ -2,6 +2,7 @@
 
 var assert = require('assert'),
     async = require('async'),
+    BoxError = require('../boxerror.js'),
     crypto = require('crypto'),
     debug = require('debug')('box:cert/acme2'),
     domains = require('../domains.js'),
@@ -24,31 +25,6 @@ exports = module.exports = {
     _getChallengeSubdomain: getChallengeSubdomain
 };
 
-function Acme2Error(reason, errorOrMessage) {
-    assert.strictEqual(typeof reason, 'string');
-    assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined');
-
-    Error.call(this);
-    Error.captureStackTrace(this, this.constructor);
-
-    this.name = this.constructor.name;
-    this.reason = reason;
-    if (typeof errorOrMessage === 'undefined') {
-        this.message = reason;
-    } else if (typeof errorOrMessage === 'string') {
-        this.message = errorOrMessage;
-    } else {
-        this.message = 'Internal error';
-        this.nestedError = errorOrMessage;
-    }
-}
-util.inherits(Acme2Error, Error);
-Acme2Error.INTERNAL_ERROR = 'Internal Error';
-Acme2Error.EXTERNAL_ERROR = 'External Error';
-Acme2Error.ALREADY_EXISTS = 'Already Exists';
-Acme2Error.NOT_COMPLETED = 'Not Completed';
-Acme2Error.FORBIDDEN = 'Forbidden';
-
 // http://jose.readthedocs.org/en/latest/
 // https://www.ietf.org/proceedings/92/slides/slides-92-acme-1.pdf
 // https://community.letsencrypt.org/t/list-of-client-implementations/2103
@@ -158,8 +134,8 @@ Acme2.prototype.updateContact = function (registrationUri, callback) {
 
     const that = this;
     this.sendSignedRequest(registrationUri, JSON.stringify(payload), function (error, result) {
-        if (error) return callback(new Acme2Error(Acme2Error.EXTERNAL_ERROR, 'Network error when registering user: ' + error.message));
-        if (result.statusCode !== 200) return callback(new Acme2Error(Acme2Error.EXTERNAL_ERROR, util.format('Failed to update contact. Expecting 200, got %s %s', result.statusCode, result.text)));
+        if (error) return callback(new BoxError(BoxError.NETWORK_ERROR, `Network error when updating contact: ${error.message}`));
+        if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Failed to update contact. Expecting 200, got %s %s', result.statusCode, result.text)));
 
         debug(`updateContact: contact of user updated to ${that.email}`);
 
@@ -178,9 +154,9 @@ Acme2.prototype.registerUser = function (callback) {
 
     var that = this;
     this.sendSignedRequest(this.directory.newAccount, JSON.stringify(payload), function (error, result) {
-        if (error) return callback(new Acme2Error(Acme2Error.EXTERNAL_ERROR, 'Network error when registering new account: ' + error.message));
+        if (error) return callback(new BoxError(BoxError.NETWORK_ERROR, `Network error when registering user: ${error.message}`));
         // 200 if already exists. 201 for new accounts
-        if (result.statusCode !== 200 && result.statusCode !== 201) return callback(new Acme2Error(Acme2Error.EXTERNAL_ERROR, util.format('Failed to register new account. Expecting 200 or 201, got %s %s', result.statusCode, result.text)));
+        if (result.statusCode !== 200 && result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Failed to register new account. Expecting 200 or 201, got %s %s', result.statusCode, result.text)));
 
         debug(`registerUser: user registered keyid: ${result.headers.location}`);
 
@@ -204,17 +180,17 @@ Acme2.prototype.newOrder = function (domain, callback) {
     debug('newOrder: %s', domain);
 
     this.sendSignedRequest(this.directory.newOrder, JSON.stringify(payload), function (error, result) {
-        if (error) return callback(new Acme2Error(Acme2Error.EXTERNAL_ERROR, 'Network error when registering domain: ' + error.message));
-        if (result.statusCode === 403) return callback(new Acme2Error(Acme2Error.FORBIDDEN, result.body.detail));
-        if (result.statusCode !== 201) return callback(new Acme2Error(Acme2Error.EXTERNAL_ERROR, util.format('Failed to register user. Expecting 201, got %s %s', result.statusCode, result.text)));
+        if (error) return callback(new BoxError(BoxError.NETWORK_ERROR, `Network error when creating new order: ${error.message}`));
+        if (result.statusCode === 403) return callback(new BoxError(BoxError.ACCESS_DENIED, `Forbidden sending signed request: ${result.body.detail}`));
+        if (result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Failed to register user. Expecting 201, got %s %s', result.statusCode, result.text)));
 
         debug('newOrder: created order %s %j', domain, result.body);
 
         const order = result.body, orderUrl = result.headers.location;
 
-        if (!Array.isArray(order.authorizations)) return callback(new Acme2Error(Acme2Error.EXTERNAL_ERROR, 'invalid authorizations in order'));
-        if (typeof order.finalize !== 'string') return callback(new Acme2Error(Acme2Error.EXTERNAL_ERROR, 'invalid finalize in order'));
-        if (typeof orderUrl !== 'string') return callback(new Acme2Error(Acme2Error.EXTERNAL_ERROR, 'invalid order location in order header'));
+        if (!Array.isArray(order.authorizations)) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'invalid authorizations in order'));
+        if (typeof order.finalize !== 'string') return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'invalid finalize in order'));
+        if (typeof orderUrl !== 'string') return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'invalid order location in order header'));
 
         callback(null, order, orderUrl);
     });
@@ -232,18 +208,18 @@ Acme2.prototype.waitForOrder = function (orderUrl, callback) {
         superagent.get(orderUrl).timeout(30 * 1000).end(function (error, result) {
             if (error && !error.response) {
                 debug('waitForOrder: network error getting uri %s', orderUrl);
-                return retryCallback(new Acme2Error(Acme2Error.EXTERNAL_ERROR, error.message)); // network error
+                return retryCallback(new BoxError(BoxError.NETWORK_ERROR, `Network error waiting for order: ${error.message}`)); // network error
             }
             if (result.statusCode !== 200) {
                 debug('waitForOrder: invalid response code getting uri %s', result.statusCode);
-                return retryCallback(new Acme2Error(Acme2Error.EXTERNAL_ERROR, 'Bad response code:' + result.statusCode));
+                return retryCallback(new BoxError(BoxError.EXTERNAL_ERROR, 'Bad response code:' + result.statusCode));
             }
 
             debug('waitForOrder: status is "%s %j', result.body.status, result.body);
 
-            if (result.body.status === 'pending' || result.body.status === 'processing') return retryCallback(new Acme2Error(Acme2Error.NOT_COMPLETED));
+            if (result.body.status === 'pending' || result.body.status === 'processing') return retryCallback(new BoxError(BoxError.TRY_AGAIN, `Request is in ${result.body.status} state`));
             else if (result.body.status === 'valid' && result.body.certificate) return retryCallback(null, result.body.certificate);
-            else return retryCallback(new Acme2Error(Acme2Error.EXTERNAL_ERROR, 'Unexpected status or invalid response: ' + result.body));
+            else return retryCallback(new BoxError(BoxError.EXTERNAL_ERROR, 'Unexpected status or invalid response: ' + result.body));
         });
     }, callback);
 };
@@ -277,8 +253,8 @@ Acme2.prototype.notifyChallengeReady = function (challenge, callback) {
     };
 
     this.sendSignedRequest(challenge.url, JSON.stringify(payload), function (error, result) {
-        if (error) return callback(new Acme2Error(Acme2Error.EXTERNAL_ERROR, 'Network error when notifying challenge: ' + error.message));
-        if (result.statusCode !== 200) return callback(new Acme2Error(Acme2Error.EXTERNAL_ERROR, util.format('Failed to notify challenge. Expecting 200, got %s %s', result.statusCode, result.text)));
+        if (error) return callback(new BoxError(BoxError.NETWORK_ERROR, `Network error when notifying challenge: ${error.message}`));
+        if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Failed to notify challenge. Expecting 200, got %s %s', result.statusCode, result.text)));
 
         callback();
     });
@@ -296,18 +272,18 @@ Acme2.prototype.waitForChallenge = function (challenge, callback) {
         superagent.get(challenge.url).timeout(30 * 1000).end(function (error, result) {
             if (error && !error.response) {
                 debug('waitForChallenge: network error getting uri %s', challenge.url);
-                return retryCallback(new Acme2Error(Acme2Error.EXTERNAL_ERROR, error.message)); // network error
+                return retryCallback(new BoxError(BoxError.NETWORK_ERROR, `Network error waiting for challenge: ${error.message}`));
             }
             if (result.statusCode !== 200) {
                 debug('waitForChallenge: invalid response code getting uri %s', result.statusCode);
-                return retryCallback(new Acme2Error(Acme2Error.EXTERNAL_ERROR, 'Bad response code:' + result.statusCode));
+                return retryCallback(new BoxError(BoxError.EXTERNAL_ERROR, 'Bad response code:' + result.statusCode));
             }
 
             debug('waitForChallenge: status is "%s %j', result.body.status, result.body);
 
-            if (result.body.status === 'pending') return retryCallback(new Acme2Error(Acme2Error.NOT_COMPLETED));
+            if (result.body.status === 'pending') return retryCallback(new BoxError(BoxError.TRY_AGAIN));
             else if (result.body.status === 'valid') return retryCallback();
-            else return retryCallback(new Acme2Error(Acme2Error.EXTERNAL_ERROR, 'Unexpected status: ' + result.body.status));
+            else return retryCallback(new BoxError(BoxError.EXTERNAL_ERROR, 'Unexpected status: ' + result.body.status));
         });
     }, function retryFinished(error) {
         // async.retry will pass 'undefined' as second arg making it unusable with async.waterfall()
@@ -329,9 +305,9 @@ Acme2.prototype.signCertificate = function (domain, finalizationUrl, csrDer, cal
     debug('signCertificate: sending sign request');
 
     this.sendSignedRequest(finalizationUrl, JSON.stringify(payload), function (error, result) {
-        if (error) return callback(new Acme2Error(Acme2Error.EXTERNAL_ERROR, 'Network error when signing certificate: ' + error.message));
+        if (error) return callback(new BoxError(BoxError.NETWORK_ERROR, `Network error when signing certificate: ${error.message}`));
         // 429 means we reached the cert limit for this domain
-        if (result.statusCode !== 200) return callback(new Acme2Error(Acme2Error.EXTERNAL_ERROR, util.format('Failed to sign certificate. Expecting 200, got %s %s', result.statusCode, result.text)));
+        if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Failed to sign certificate. Expecting 200, got %s %s', result.statusCode, result.text)));
 
         return callback(null);
     });
@@ -351,15 +327,15 @@ Acme2.prototype.createKeyAndCsr = function (hostname, callback) {
         debug('createKeyAndCsr: reuse the key for renewal at %s', privateKeyFile);
     } else {
         var key = safe.child_process.execSync('openssl genrsa 4096');
-        if (!key) return callback(new Acme2Error(Acme2Error.INTERNAL_ERROR, safe.error));
-        if (!safe.fs.writeFileSync(privateKeyFile, key)) return callback(new Acme2Error(Acme2Error.INTERNAL_ERROR, safe.error));
+        if (!key) return callback(new BoxError(BoxError.OPENSSL_ERROR, safe.error));
+        if (!safe.fs.writeFileSync(privateKeyFile, key)) return callback(new BoxError(BoxError.FS_ERROR, safe.error));
 
         debug('createKeyAndCsr: key file saved at %s', privateKeyFile);
     }
 
     var csrDer = safe.child_process.execSync(`openssl req -new -key ${privateKeyFile} -outform DER -subj /CN=${hostname}`);
-    if (!csrDer) return callback(new Acme2Error(Acme2Error.INTERNAL_ERROR, safe.error));
-    if (!safe.fs.writeFileSync(csrFile, csrDer)) return callback(new Acme2Error(Acme2Error.INTERNAL_ERROR, safe.error)); // bookkeeping
+    if (!csrDer) return callback(new BoxError(BoxError.OPENSSL_ERROR, safe.error));
+    if (!safe.fs.writeFileSync(csrFile, csrDer)) return callback(new BoxError(BoxError.FS_ERROR, safe.error)); // bookkeeping
 
     debug('createKeyAndCsr: csr file (DER) saved at %s', csrFile);
 
@@ -373,25 +349,29 @@ Acme2.prototype.downloadCertificate = function (hostname, certUrl, callback) {
 
     var outdir = paths.APP_CERTS_DIR;
 
-    superagent.get(certUrl).buffer().parse(function (res, done) {
-        var data = [ ];
-        res.on('data', function(chunk) { data.push(chunk); });
-        res.on('end', function () { res.text = Buffer.concat(data); done(); });
-    }).timeout(30 * 1000).end(function (error, result) {
-        if (error && !error.response) return callback(new Acme2Error(Acme2Error.EXTERNAL_ERROR, 'Network error when downloading certificate'));
-        if (result.statusCode === 202) return callback(new Acme2Error(Acme2Error.INTERNAL_ERROR, 'Retry not implemented yet'));
-        if (result.statusCode !== 200) return callback(new Acme2Error(Acme2Error.EXTERNAL_ERROR, util.format('Failed to get cert. Expecting 200, got %s %s', result.statusCode, result.text)));
+    async.retry({ times: 5, interval: 20000 }, function (retryCallback) {
+        debug('downloadCertificate: downloading certificate');
 
-        const fullChainPem = result.text;
+        superagent.get(certUrl).buffer().parse(function (res, done) {
+            var data = [ ];
+            res.on('data', function(chunk) { data.push(chunk); });
+            res.on('end', function () { res.text = Buffer.concat(data); done(); });
+        }).timeout(30 * 1000).end(function (error, result) {
+            if (error && !error.response) return retryCallback(new BoxError(BoxError.NETWORK_ERROR, `Network error when downloading certificate: ${error.message}`));
+            if (result.statusCode === 202) return retryCallback(new BoxError(BoxError.TRY_AGAIN, 'Retry'));
+            if (result.statusCode !== 200) return retryCallback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Failed to get cert. Expecting 200, got %s %s', result.statusCode, result.text)));
 
-        const certName = hostname.replace('*.', '_.');
-        var certificateFile = path.join(outdir, `${certName}.cert`);
-        if (!safe.fs.writeFileSync(certificateFile, fullChainPem)) return callback(new Acme2Error(Acme2Error.INTERNAL_ERROR, safe.error));
+            const fullChainPem = result.text;
 
-        debug('downloadCertificate: cert file for %s saved at %s', hostname, certificateFile);
+            const certName = hostname.replace('*.', '_.');
+            var certificateFile = path.join(outdir, `${certName}.cert`);
+            if (!safe.fs.writeFileSync(certificateFile, fullChainPem)) return retryCallback(new BoxError(BoxError.FS_ERROR, safe.error));
 
-        callback();
-    });
+            debug('downloadCertificate: cert file for %s saved at %s', hostname, certificateFile);
+
+            retryCallback(null);
+        });
+    }, callback);
 };
 
 Acme2.prototype.prepareHttpChallenge = function (hostname, domain, authorization, callback) {
@@ -402,7 +382,7 @@ Acme2.prototype.prepareHttpChallenge = function (hostname, domain, authorization
 
     debug('acmeFlow: challenges: %j', authorization);
     let httpChallenges = authorization.challenges.filter(function(x) { return x.type === 'http-01'; });
-    if (httpChallenges.length === 0) return callback(new Acme2Error(Acme2Error.EXTERNAL_ERROR, 'no http challenges'));
+    if (httpChallenges.length === 0) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'no http challenges'));
     let challenge = httpChallenges[0];
 
     debug('prepareHttpChallenge: preparing for challenge %j', challenge);
@@ -412,7 +392,7 @@ Acme2.prototype.prepareHttpChallenge = function (hostname, domain, authorization
     debug('prepareHttpChallenge: writing %s to %s', keyAuthorization, path.join(paths.ACME_CHALLENGES_DIR, challenge.token));
 
     fs.writeFile(path.join(paths.ACME_CHALLENGES_DIR, challenge.token), keyAuthorization, function (error) {
-        if (error) return callback(new Acme2Error(Acme2Error.INTERNAL_ERROR, error));
+        if (error) return callback(new BoxError(BoxError.FS_ERROR, error));
 
         callback(null, challenge);
     });
@@ -454,7 +434,7 @@ Acme2.prototype.prepareDnsChallenge = function (hostname, domain, authorization,
 
     debug('acmeFlow: challenges: %j', authorization);
     let dnsChallenges = authorization.challenges.filter(function(x) { return x.type === 'dns-01'; });
-    if (dnsChallenges.length === 0) return callback(new Acme2Error(Acme2Error.EXTERNAL_ERROR, 'no dns challenges'));
+    if (dnsChallenges.length === 0) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'no dns challenges'));
     let challenge = dnsChallenges[0];
 
     const keyAuthorization = this.getKeyAuthorization(challenge.token);
@@ -467,10 +447,10 @@ Acme2.prototype.prepareDnsChallenge = function (hostname, domain, authorization,
     debug(`prepareDnsChallenge: update ${challengeSubdomain} with ${txtValue}`);
 
     domains.upsertDnsRecords(challengeSubdomain, domain, 'TXT', [ `"${txtValue}"` ], function (error) {
-        if (error) return callback(new Acme2Error(Acme2Error.EXTERNAL_ERROR, error.message));
+        if (error) return callback(error);
 
         domains.waitForDnsRecord(challengeSubdomain, domain, 'TXT', txtValue, { interval: 5000, times: 200 }, function (error) {
-            if (error) return callback(new Acme2Error(Acme2Error.EXTERNAL_ERROR, error.message));
+            if (error) return callback(error);
 
             callback(null, challenge);
         });
@@ -493,7 +473,7 @@ Acme2.prototype.cleanupDnsChallenge = function (hostname, domain, challenge, cal
     debug(`cleanupDnsChallenge: remove ${challengeSubdomain} with ${txtValue}`);
 
     domains.removeDnsRecords(challengeSubdomain, domain, 'TXT', [ `"${txtValue}"` ], function (error) {
-        if (error) return callback(new Acme2Error(Acme2Error.EXTERNAL_ERROR, error));
+        if (error) return callback(error);
 
         callback(null);
     });
@@ -505,10 +485,12 @@ Acme2.prototype.prepareChallenge = function (hostname, domain, authorizationUrl,
     assert.strictEqual(typeof authorizationUrl, 'string');
     assert.strictEqual(typeof callback, 'function');
 
+    debug(`prepareChallenge: http: ${this.performHttpAuthorization}`);
+
     const that = this;
     superagent.get(authorizationUrl).timeout(30 * 1000).end(function (error, response) {
-        if (error && !error.response) return callback(error);
-        if (response.statusCode !== 200) return callback(new Error('Invalid response code getting authorization : ' + response.statusCode));
+        if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, `Network error when preparing challenge: ${error.message}`));
+        if (response.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response code getting authorization : ' + response.statusCode));
 
         const authorization = response.body;
 
@@ -526,6 +508,8 @@ Acme2.prototype.cleanupChallenge = function (hostname, domain, challenge, callba
     assert.strictEqual(typeof challenge, 'object');
     assert.strictEqual(typeof callback, 'function');
 
+    debug(`cleanupChallenge: http: ${this.performHttpAuthorization}`);
+
     if (this.performHttpAuthorization) {
         this.cleanupHttpChallenge(hostname, domain, challenge, callback);
     } else {
@@ -541,7 +525,7 @@ Acme2.prototype.acmeFlow = function (hostname, domain, callback) {
     if (!fs.existsSync(paths.ACME_ACCOUNT_KEY_FILE)) {
         debug('getCertificate: generating acme account key on first run');
         this.accountKeyPem = safe.child_process.execSync('openssl genrsa 4096');
-        if (!this.accountKeyPem) return callback(new Acme2Error(Acme2Error.INTERNAL_ERROR, safe.error));
+        if (!this.accountKeyPem) return callback(new BoxError(BoxError.OPENSSL_ERROR, safe.error));
 
         safe.fs.writeFileSync(paths.ACME_ACCOUNT_KEY_FILE, this.accountKeyPem);
     } else {
@@ -586,8 +570,8 @@ Acme2.prototype.getDirectory = function (callback) {
     const that = this;
 
     superagent.get(this.caDirectory).timeout(30 * 1000).end(function (error, response) {
-        if (error && !error.response) return callback(error);
-        if (response.statusCode !== 200) return callback(new Error('Invalid response code when fetching directory : ' + response.statusCode));
+        if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, `Network error getting directory: ${error.message}`));
+        if (response.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response code when fetching directory : ' + response.statusCode));
 
         if (typeof response.body.newNonce !== 'string' ||
             typeof response.body.newOrder !== 'string' ||
@@ -631,6 +615,11 @@ function getCertificate(hostname, domain, options, callback) {
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    var acme = new Acme2(options || { });
-    acme.getCertificate(hostname, domain, callback);
+    let attempt = 1;
+    async.retry({ times: 3, interval: 0 }, function (retryCallback) {
+        debug(`getCertificate: attempt ${attempt++}`);
+
+        let acme = new Acme2(options || { });
+        acme.getCertificate(hostname, domain, retryCallback);
+    }, callback);
 }

src/cloudron.js

@@ -135,7 +135,7 @@ function notifyUpdate(callback) {
 // each of these tasks can fail. we will add some routes to fix/re-run them
 function runStartupTasks() {
     // configure nginx to be reachable by IP
-    reverseProxy.configureDefaultServer(NOOP_CALLBACK);
+    reverseProxy.writeDefaultConfig(NOOP_CALLBACK);
 
     // always generate webadmin config since we have no versioning mechanism for the ejs
     if (settings.adminDomain()) reverseProxy.writeAdminConfig(settings.adminDomain(), NOOP_CALLBACK);

src/custom.js

@@ -38,7 +38,7 @@ const DEFAULT_SPEC = {
     },
     alerts: {
         email: '',
-        notifyCloudronAdmins: false
+        notifyCloudronAdmins: true
     },
     footer: {
         body: '&copy; 2019 [Cloudron](https://cloudron.io) [Forum <i class="fa fa-comments"></i>](https://forum.cloudron.io)'

src/dns/digitalocean.js

@@ -66,12 +66,10 @@ function getInternal(dnsConfig, zoneName, name, type, callback) {
 
                 nextPage = (result.body.links && result.body.links.pages) ? result.body.links.pages.next : null;
 
-                debug(`getInternal: next page - ${nextPage}`);
-
                 iteratorDone();
             });
     }, function () { return !!nextPage; }, function (error) {
-        debug('getInternal:', error, matchingRecords);
+        debug('getInternal:', error, JSON.stringify(matchingRecords));
 
         if (error) return callback(error);
 

src/docker.js

@@ -180,7 +180,7 @@ function createSubcontainer(app, name, cmd, options, callback) {
     var portEnv = [];
     for (let portName in app.portBindings) {
         const hostPort = app.portBindings[portName];
-        const portType = portName in manifest.tcpPorts ? 'tcp' : 'udp';
+        const portType = (manifest.tcpPorts && portName in manifest.tcpPorts) ? 'tcp' : 'udp';
         const ports = portType == 'tcp' ? manifest.tcpPorts : manifest.udpPorts;
 
         var containerPort = ports[portName].containerPort || hostPort;

src/eventlog.js

@@ -26,7 +26,7 @@ exports = module.exports = {
 
     ACTION_BACKUP_FINISH: 'backup.finish',
     ACTION_BACKUP_START: 'backup.start',
-    ACTION_BACKUP_CLEANUP_START: 'backup.cleanup.start',
+    ACTION_BACKUP_CLEANUP_START: 'backup.cleanup.start', // obsolete
     ACTION_BACKUP_CLEANUP_FINISH: 'backup.cleanup.finish',
 
     ACTION_CERTIFICATE_NEW: 'certificate.new',

src/notifications.js

@@ -24,7 +24,7 @@ exports = module.exports = {
 
 let assert = require('assert'),
     async = require('async'),
-    auditsource = require('./auditsource.js'),
+    auditSource = require('./auditsource.js'),
     changelog = require('./changelog.js'),
     custom = require('./custom.js'),
     DatabaseError = require('./databaseerror.js'),
@@ -294,7 +294,7 @@ function backupFailed(eventId, taskId, errorMessage, callback) {
 
     actionForAllAdmins([], function (admin, callback) {
         mailer.backupFailed(admin.email, errorMessage, `${settings.adminOrigin()}/logs.html?taskId=${taskId}`);
-        add(admin.id, eventId, 'Failed to backup', `Backup failed: ${errorMessage}. Logs are available [here](/logs.html?taskId=${taskId}). Will be retried in 4 hours`, callback);
+        add(admin.id, eventId, 'Backup failed', `Backup failed: ${errorMessage}. Logs are available [here](/logs.html?taskId=${taskId}). Will be retried in 4 hours`, callback);
     }, callback);
 }
 
@@ -347,7 +347,7 @@ function onEvent(id, action, source, data, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     // external ldap syncer does not generate notifications - FIXME username might be an issue here
-    if (source.username === auditsource.EXTERNAL_LDAP_TASK.username) return callback();
+    if (source.username === auditSource.EXTERNAL_LDAP_TASK.username) return callback();
 
     switch (action) {
     case eventlog.ACTION_USER_ADD:
@@ -370,6 +370,7 @@ function onEvent(id, action, source, data, callback) {
         return appUp(id, data.app, callback);
 
     case eventlog.ACTION_APP_UPDATE_FINISH:
+        if (!data.app.appStoreId) return callback(); // skip notification of dev apps
         return appUpdated(id, data.app, callback);
 
     case eventlog.ACTION_CERTIFICATE_RENEWAL:
@@ -378,8 +379,10 @@ function onEvent(id, action, source, data, callback) {
         return certificateRenewalError(id, data.domain, data.errorMessage, callback);
 
     case eventlog.ACTION_BACKUP_FINISH:
-        if (!data.errorMessage || source.username !== 'cron') return callback();
-        return backupFailed(id, data.taskId, data.errorMessage, callback); // only notify for automated backups
+        if (!data.errorMessage) return callback();
+        if (source.username !== auditSource.CRON.username && !data.timedOut) return callback(); // manual stop by user
+
+        return backupFailed(id, data.taskId, data.errorMessage, callback); // only notify for automated backups or timedout
 
     case eventlog.ACTION_UPDATE_FINISH:
         return boxUpdated(data.oldVersion, data.newVersion, callback);

src/reverseproxy.js

@@ -16,16 +16,16 @@ exports = module.exports = {
 
     renewCerts: renewCerts,
 
-    // the 'configure' functions always ensure a certificate
-    configureDefaultServer: configureDefaultServer,
+    // the 'configure' ensure a certificate and generate nginx config
     configureAdmin: configureAdmin,
     configureApp: configureApp,
     unconfigureApp: unconfigureApp,
 
+    // these only generate nginx config
+    writeDefaultConfig: writeDefaultConfig,
     writeAdminConfig: writeAdminConfig,
     writeAppConfig: writeAppConfig,
 
-    reload: reload,
     removeAppConfigs: removeAppConfigs,
 
     // exported for testing
@@ -261,13 +261,13 @@ function getFallbackCertificate(domain, callback) {
     var certFilePath = path.join(paths.NGINX_CERT_DIR, `${domain}.host.cert`);
     var keyFilePath = path.join(paths.NGINX_CERT_DIR, `${domain}.host.key`);
 
-    if (fs.existsSync(certFilePath) && fs.existsSync(keyFilePath)) return callback(null, { certFilePath, keyFilePath, type: 'provisioned' });
+    if (fs.existsSync(certFilePath) && fs.existsSync(keyFilePath)) return callback(null, { certFilePath, keyFilePath });
 
     // check for auto-generated or user set fallback certs
     certFilePath = path.join(paths.APP_CERTS_DIR, `${domain}.host.cert`);
     keyFilePath = path.join(paths.APP_CERTS_DIR, `${domain}.host.key`);
 
-    callback(null, { certFilePath, keyFilePath, type: 'fallback' });
+    callback(null, { certFilePath, keyFilePath });
 }
 
 function setAppCertificateSync(location, domainObject, certificate) {
@@ -333,6 +333,8 @@ function notifyCertChanged(vhost, callback) {
     assert.strictEqual(typeof vhost, 'string');
     assert.strictEqual(typeof callback, 'function');
 
+    debug(`notifyCertChanged: vhost: ${vhost} mailFqdn: ${settings.mailFqdn()}`);
+
     if (vhost !== settings.mailFqdn()) return callback();
 
     mail.handleCertChanged(callback);
@@ -350,12 +352,12 @@ function ensureCertificate(vhost, domain, auditSource, callback) {
         getCertApi(domainObject, function (error, api, apiOptions) {
             if (error) return callback(error);
 
-            getCertificateByHostname(vhost, domainObject, function (error, currentBundle) {
+            getCertificateByHostname(vhost, domainObject, function (_error, currentBundle) {
                 if (currentBundle) {
                     debug(`ensureCertificate: ${vhost} certificate already exists at ${currentBundle.keyFilePath}`);
 
-                    if (currentBundle.certFilePath.endsWith('.user.cert')) return callback(null, currentBundle); // user certs cannot be renewed
-                    if (!isExpiringSync(currentBundle.certFilePath, 24 * 30) && providerMatchesSync(domainObject, currentBundle.certFilePath, apiOptions)) return callback(null, currentBundle);
+                    if (currentBundle.certFilePath.endsWith('.user.cert')) return callback(null, currentBundle, { renewed: false }); // user certs cannot be renewed
+                    if (!isExpiringSync(currentBundle.certFilePath, 24 * 30) && providerMatchesSync(domainObject, currentBundle.certFilePath, apiOptions)) return callback(null, currentBundle, { renewed: false });
                     debug(`ensureCertificate: ${vhost} cert require renewal`);
                 } else {
                     debug(`ensureCertificate: ${vhost} cert does not exist`);
@@ -364,15 +366,28 @@ function ensureCertificate(vhost, domain, auditSource, callback) {
                 debug('ensureCertificate: getting certificate for %s with options %j', vhost, apiOptions);
 
                 api.getCertificate(vhost, domain, apiOptions, function (error, certFilePath, keyFilePath) {
+                    debug(`ensureCertificate: error: ${error ? error.message : 'null'} cert: ${certFilePath}`);
+
                     eventlog.add(currentBundle ? eventlog.ACTION_CERTIFICATE_RENEWAL : eventlog.ACTION_CERTIFICATE_NEW, auditSource, { domain: vhost, errorMessage: error ? error.message : '' });
 
+                    if (error && currentBundle && !isExpiringSync(currentBundle.certFilePath, 0)) {
+                        debug('ensureCertificate: continue using existing bundle since renewal failed');
+                        return callback(null, currentBundle, { renewed: false });
+                    }
+
                     notifyCertChanged(vhost, function (error) {
                         if (error) return callback(error);
 
-                        // if no cert was returned use fallback. the fallback/caas provider will not provide any for example
-                        if (!certFilePath || !keyFilePath) return getFallbackCertificate(domain, callback);
+                        if (certFilePath && keyFilePath) return callback(null, { certFilePath, keyFilePath }, { renewed: true });
 
-                        callback(null, { certFilePath, keyFilePath });
+                        debug(`ensureCertificate: renewal of ${vhost} failed. using fallback certificates for ${domain}`);
+
+                        // if no cert was returned use fallback. the fallback/caas provider will not provide any for example
+                        getFallbackCertificate(domain, function (error, bundle) {
+                            if (error) return callback(error);
+
+                            callback(null, bundle, { renewed: false });
+                        });
                     });
                 });
             });
@@ -401,8 +416,6 @@ function writeAdminNginxConfig(bundle, configFileName, vhost, callback) {
 
     if (!safe.fs.writeFileSync(nginxConfigFilename, nginxConf)) return callback(safe.error);
 
-    if (vhost) safe.fs.unlinkSync(path.join(paths.NGINX_APPCONFIG_DIR, 'admin.conf')); // remove legacy admin.conf. remove after 3.5
-
     reload(callback);
 }
 
@@ -428,6 +441,8 @@ function writeAdminConfig(domain, callback) {
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof callback, 'function');
 
+    debug(`writeAdminConfig: writing admin config for ${domain}`);
+
     domains.get(domain, function (error, domainObject) {
         if (error) return callback(error);
 
@@ -463,7 +478,7 @@ function writeAppNginxConfig(app, bundle, callback) {
     var nginxConf = ejs.render(NGINX_APPCONFIG_EJS, data);
 
     var nginxConfigFilename = path.join(paths.NGINX_APPCONFIG_DIR, app.id + '.conf');
-    debug('writing config for "%s" to %s with options %j', app.fqdn, nginxConfigFilename, data);
+    debug('writeAppNginxConfig: writing config for "%s" to %s with options %j', app.fqdn, nginxConfigFilename, data);
 
     if (!safe.fs.writeFileSync(nginxConfigFilename, nginxConf)) {
         debug('Error creating nginx config for "%s" : %s', app.fqdn, safe.error.message);
@@ -584,14 +599,17 @@ function renewCerts(options, auditSource, progressCallback, callback) {
 
         if (options.domain) appDomains = appDomains.filter(function (appDomain) { return appDomain.domain === options.domain; });
 
-        let progress = 1;
+        let progress = 1, renewed = [];
+
         async.eachSeries(appDomains, function (appDomain, iteratorCallback) {
             progressCallback({ percent: progress, message: `Renewing certs of ${appDomain.fqdn}` });
             progress += Math.round(100/appDomains.length);
 
-            ensureCertificate(appDomain.fqdn, appDomain.domain, auditSource, function (error, bundle) {
+            ensureCertificate(appDomain.fqdn, appDomain.domain, auditSource, function (error, bundle, state) {
                 if (error) return iteratorCallback(error); // this can happen if cloudron is not setup yet
 
+                if (state.renewed) renewed.push(appDomain.fqdn);
+
                 // hack to check if the app's cert changed or not. this doesn't handle prod/staging le change since they use same file name
                 let currentNginxConfig = safe.fs.readFileSync(appDomain.nginxConfigFilename, 'utf8') || '';
                 if (currentNginxConfig.includes(bundle.certFilePath)) return iteratorCallback();
@@ -607,7 +625,15 @@ function renewCerts(options, auditSource, progressCallback, callback) {
 
                 configureFunc(iteratorCallback);
             });
-        }, callback);
+        }, function (error) {
+            if (error) return callback(error);
+
+            debug(`renewCerts: Renewed certs of ${JSON.stringify(renewed)}`);
+            if (renewed.length === 0) return callback(null);
+
+            // reload nginx if any certs were updated but the config was not rewritten
+            reload(callback);
+        });
     });
 }
 
@@ -619,18 +645,18 @@ function removeAppConfigs() {
     }
 }
 
-function configureDefaultServer(callback) {
+function writeDefaultConfig(callback) {
     assert.strictEqual(typeof callback, 'function');
 
     var certFilePath = path.join(paths.NGINX_CERT_DIR,  'default.cert');
     var keyFilePath = path.join(paths.NGINX_CERT_DIR, 'default.key');
 
     if (!fs.existsSync(certFilePath) || !fs.existsSync(keyFilePath)) {
-        debug('configureDefaultServer: create new cert');
+        debug('writeDefaultConfig: create new cert');
 
         var cn = 'cloudron-' + (new Date()).toISOString(); // randomize date a bit to keep firefox happy
         if (!safe.child_process.execSync(`openssl req -x509 -newkey rsa:2048 -keyout ${keyFilePath} -out ${certFilePath} -days 3650 -subj /CN=${cn} -nodes`)) {
-            debug(`configureDefaultServer: could not generate certificate: ${safe.error.message}`);
+            debug(`writeDefaultConfig: could not generate certificate: ${safe.error.message}`);
             return callback(safe.error);
         }
     }
@@ -638,7 +664,7 @@ function configureDefaultServer(callback) {
     writeAdminNginxConfig({ certFilePath, keyFilePath }, constants.NGINX_DEFAULT_CONFIG_FILE_NAME, '', function (error) {
         if (error) return callback(error);
 
-        debug('configureDefaultServer: done');
+        debug('writeDefaultConfig: done');
 
         callback(null);
     });

src/routes/apps.js

@@ -328,8 +328,7 @@ function setLocation(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
     assert.strictEqual(typeof req.params.id, 'string');
 
-    if (!req.body.location) return next(new HttpError(400, 'location is required'));
-    if (typeof req.body.location !== 'string') return next(new HttpError(400, 'location must be string'));
+    if (typeof req.body.location !== 'string') return next(new HttpError(400, 'location must be string')); // location may be an empty string
     if (!req.body.domain) return next(new HttpError(400, 'domain is required'));
     if (typeof req.body.domain !== 'string') return next(new HttpError(400, 'domain must be string'));
 
@@ -489,6 +488,7 @@ function updateApp(req, res, next) {
     if ('appStoreId' in data && typeof data.appStoreId !== 'string') return next(new HttpError(400, 'appStoreId must be a string'));
     if (!data.manifest && !data.appStoreId) return next(new HttpError(400, 'appStoreId or manifest is required'));
 
+    if ('skipBackup' in data && typeof data.skipBackup !== 'boolean') return next(new HttpError(400, 'skipBackup must be a boolean'));
     if ('force' in data && typeof data.force !== 'boolean') return next(new HttpError(400, 'force must be a boolean'));
 
     debug('Update app id:%s to manifest:%j', req.params.id, data.manifest);

src/routes/test/apps-test.js

@@ -330,7 +330,7 @@ describe('App API', function () {
                 .query({ access_token: token })
                 .send({ manifest: APP_MANIFEST, location: 'some', accessRestriction: null, domain: 'doesnotexist.com' })
                 .end(function (err, res) {
-                    expect(res.statusCode).to.equal(404);
+                    expect(res.statusCode).to.equal(400);
                     expect(res.body.message).to.eql('No such domain');
                     done();
                 });

src/storage/s3.js

@@ -95,7 +95,10 @@ function upload(apiConfig, backupFilePath, sourceStream, callback) {
 
         // s3.upload automatically does a multi-part upload. we set queueSize to 1 to reduce memory usage
         // uploader will buffer at most queueSize * partSize bytes into memory at any given time.
-        s3.upload(params, { partSize: 10 * 1024 * 1024, queueSize: 1 }, function (error, data) {
+        // scaleway only supports 1000 parts per object (https://www.scaleway.com/en/docs/s3-multipart-upload/)
+        const partSize = apiConfig.provider === 'scaleway-objectstorage' ? 100 * 1024 * 1024 : 10 * 1024 * 1024;
+
+        s3.upload(params, { partSize, queueSize: 1 }, function (error, data) {
             if (error) {
                 debug('Error uploading [%s]: s3 upload error.', backupFilePath, error);
                 return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, `Error uploading ${backupFilePath}. Message: ${error.message} HTTP Code: ${error.code}`));

src/tasks.js

@@ -30,6 +30,7 @@ exports = module.exports = {
     // error codes
     ESTOPPED: 'stopped',
     ECRASHED: 'crashed',
+    ETIMEOUT: 'timeout',
 
     // testing
     _TASK_IDENTITY: '_identity',
@@ -165,7 +166,7 @@ function startTask(taskId, options, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     const logFile = options.logFile || `${paths.TASKS_LOG_DIR}/${taskId}.log`;
-    let fd = safe.fs.openSync(logFile, 'w'); // will autoclose
+    let fd = safe.fs.openSync(logFile, 'a'); // will autoclose. append is for apptask logs
     if (!fd) {
         debug(`startTask: unable to get log filedescriptor ${safe.error.message}`);
         return callback(new TaskError(TaskError.INTERNAL_ERROR, safe.error));
@@ -173,16 +174,20 @@ function startTask(taskId, options, callback) {
 
     debug(`startTask - starting task ${taskId}. logs at ${logFile}`);
 
+    let killTimerId = null, timedOut = false;
+
     gTasks[taskId] = child_process.fork(`${__dirname}/taskworker.js`, [ taskId ], { stdio: [ 'pipe', fd, fd, 'ipc' ]}); // fork requires ipc
     gTasks[taskId].once('exit', function (code, signal) {
         debug(`startTask: ${taskId} completed with code ${code} and signal ${signal}`);
 
+        if (options.timeout) clearTimeout(killTimerId);
+
         get(taskId, function (error, task) {
             let taskError;
             if (!error && task.percent !== 100) { // task crashed or was killed by us
                 taskError = {
-                    message: code === 0 ? `Task ${taskId} stopped` : `Task ${taskId} crashed with code ${code} and signal ${signal}`,
-                    code: code === 0 ? exports.ESTOPPED : exports.ECRASHED
+                    message: code === 0 ? `Task ${taskId} ${timedOut ? 'timed out' : 'stopped'}` : `Task ${taskId} crashed with code ${code} and signal ${signal}`,
+                    code: code === 0 ? (timedOut ? exports.ETIMEOUT : exports.ESTOPPED) : exports.ECRASHED
                 };
                 // note that despite the update() here, we should handle the case where the box code was restarted and never got taskworker exit
                 setCompleted(taskId, { error: taskError }, NOOP_CALLBACK);
@@ -199,6 +204,14 @@ function startTask(taskId, options, callback) {
             debug(`startTask: ${taskId} done`);
         });
     });
+
+    if (options.timeout) {
+        killTimerId = setTimeout(function () {
+            debug(`startTask: task ${taskId} took too long. killing`);
+            timedOut = true;
+            stopTask(taskId, NOOP_CALLBACK);
+        }, options.timeout);
+    }
 }
 
 function stopTask(id, callback) {

src/taskworker.js

@@ -47,7 +47,7 @@ function initialize(callback) {
 }
 
 // Main process starts here
-debug(`Staring task ${taskId}`);
+debug(`Starting task ${taskId}`);
 
 initialize(function (error) {
     if (error) return process.exit(50);

src/test/apps-test.js

@@ -407,9 +407,9 @@ describe('Apps', function () {
                 expect(error).to.be(null);
 
                 apps.getAll(function (error, result) {
-                    expect(result[0].installationState).to.be(apps.ISTATE_PENDING_RESTORE);
+                    expect(result[0].installationState).to.be(apps.ISTATE_PENDING_INSTALL);
                     expect(result[1].installationState).to.be(apps.ISTATE_ERROR);
-                    expect(result[2].installationState).to.be(apps.ISTATE_PENDING_RESTORE);
+                    expect(result[2].installationState).to.be(apps.ISTATE_PENDING_INSTALL);
 
                     done();
                 });