Add a timeout for update as well

this will send a notification if an update gets stuck

src/cloudron.js

@@ -119,7 +119,7 @@ function notifyUpdate(callback) {
     const version = safe.fs.readFileSync(paths.VERSION_FILE, 'utf8');
     if (version === constants.VERSION) return callback();
 
-    eventlog.add(eventlog.ACTION_UPDATE_FINISH, auditSource.CRON, { oldVersion: version || 'dev', newVersion: constants.VERSION }, function (error) {
+    eventlog.add(eventlog.ACTION_UPDATE_FINISH, auditSource.CRON, { errorMessage: '', oldVersion: version || 'dev', newVersion: constants.VERSION }, function (error) {
         if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
 
         tasks.setCompletedByType(tasks.TASK_UPDATE, { error: null }, function (error) {

src/mail_templates/box_update_error.ejs

@@ -0,0 +1,20 @@
+<%if (format === 'text') { %>
+
+Dear Cloudron Admin,
+
+Cloudron update failed because of the following reason:
+
+-------------------------------------
+
+<%- message %>
+
+-------------------------------------
+
+
+Powered by https://cloudron.io
+
+Sent at: <%= new Date().toUTCString() %>
+
+<% } else { %>
+
+<% } %>

src/mailer.js

@@ -17,6 +17,7 @@ exports = module.exports = {
     backupFailed: backupFailed,
 
     certificateRenewalError: certificateRenewalError,
+    boxUpdateError: boxUpdateError,
 
     sendTestMail: sendTestMail,
 
@@ -411,6 +412,24 @@ function certificateRenewalError(mailTo, domain, message) {
     });
 }
 
+function boxUpdateError(mailTo, message) {
+    assert.strictEqual(typeof mailTo, 'string');
+    assert.strictEqual(typeof message, 'string');
+
+    getMailConfig(function (error, mailConfig) {
+        if (error) return debug('Error getting mail details:', error);
+
+        var mailOptions = {
+            from: mailConfig.notificationFrom,
+            to: mailTo,
+            subject: util.format('[%s] Cloudron update error', mailConfig.cloudronName),
+            text: render('box_update_error.ejs', { message: message, format: 'text' })
+        };
+
+        sendMail(mailOptions);
+    });
+}
+
 function oomEvent(mailTo, program, event) {
     assert.strictEqual(typeof mailTo, 'string');
     assert.strictEqual(typeof program, 'string');

src/notifications.js

@@ -255,7 +255,8 @@ function appUpdated(eventId, app, callback) {
     }, callback);
 }
 
-function boxUpdated(oldVersion, newVersion, callback) {
+function boxUpdated(eventId, oldVersion, newVersion, callback) {
+    assert.strictEqual(typeof eventId, 'string');
     assert.strictEqual(typeof oldVersion, 'string');
     assert.strictEqual(typeof newVersion, 'string');
     assert.strictEqual(typeof callback, 'function');
@@ -264,7 +265,21 @@ function boxUpdated(oldVersion, newVersion, callback) {
     const changelogMarkdown = changes.map((m) => `* ${m}\n`).join('');
 
     actionForAllAdmins([], function (admin, done) {
-        add(admin.id, null, `Cloudron updated to v${newVersion}`, `Cloudron was updated from v${oldVersion} to v${newVersion}.\n\nChangelog:\n${changelogMarkdown}\n`, done);
+        add(admin.id, eventId, `Cloudron updated to v${newVersion}`, `Cloudron was updated from v${oldVersion} to v${newVersion}.\n\nChangelog:\n${changelogMarkdown}\n`, done);
+    }, callback);
+}
+
+function boxUpdateError(eventId, errorMessage, callback) {
+    assert.strictEqual(typeof eventId, 'string');
+    assert.strictEqual(typeof errorMessage, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    if (custom.spec().alerts.email) mailer.boxUpdateError(custom.spec().alerts.email, errorMessage);
+    if (!custom.spec().alerts.notifyCloudronAdmins) return callback();
+
+    actionForAllAdmins([], function (admin, done) {
+        mailer.boxUpdateError(admin.email, errorMessage);
+        add(admin.id, eventId, 'Cloudron update failed', `Failed to update Cloudron: ${errorMessage}. Update will be retried in 4 hours`, done);
     }, callback);
 }
 
@@ -385,7 +400,9 @@ function onEvent(id, action, source, data, callback) {
         return backupFailed(id, data.taskId, data.errorMessage, callback); // only notify for automated backups or timedout
 
     case eventlog.ACTION_UPDATE_FINISH:
-        return boxUpdated(data.oldVersion, data.newVersion, callback);
+        if (!data.errorMessage) return boxUpdated(id, data.oldVersion, data.newVersion, callback);
+        if (data.timedOut) return boxUpdateError(id, data.errorMessage, callback);
+        return callback();
 
     default:
         return callback();

src/updater.js

@@ -249,10 +249,13 @@ function updateToLatest(options, auditSource, callback) {
 
             eventlog.add(eventlog.ACTION_UPDATE, auditSource, { taskId, boxUpdateInfo });
 
-            tasks.startTask(taskId, {}, (error) => {
+            tasks.startTask(taskId, { timeout: 20 * 60 * 60 * 1000 /* 20 hours */ }, (error) => {
                 locker.unlock(locker.OP_BOX_UPDATE);
 
                 debug('Update failed with error', error);
+
+                const timedOut = error.code === tasks.ETIMEOUT;
+                eventlog.add(eventlog.ACTION_UPDATE_FINISH, auditSource, { taskId, errorMessage: error.message, timedOut });
             });
 
             callback(null, taskId);