diff --git a/CHANGES b/CHANGES
index c0723d27b..138094366 100644
--- a/CHANGES
+++ b/CHANGES
@@ -3180,4 +3180,5 @@
* backup logs: make them much terse and concise
* oidc: implement Device Authorization Grant
* operator: fix viewing of backup progress and logs
+* notification: automatic app update failure notification
diff --git a/dashboard/public/translation/en.json b/dashboard/public/translation/en.json
index 06a6ee42f..365f1d745 100644
--- a/dashboard/public/translation/en.json
+++ b/dashboard/public/translation/en.json
@@ -890,7 +890,8 @@
"appDown": "App is down",
"rebootRequired": "Server reboot required",
"cloudronUpdateFailed": "Cloudron update failed",
- "diskSpace": "Low disk space"
+ "diskSpace": "Low disk space",
+ "appAutoUpdateFailed": "App automatic update failed"
},
"settingsDialog": {
"description": "An email will be sent for the selected events to your primary email."
diff --git a/dashboard/src/components/NotificationSettingsDialog.vue b/dashboard/src/components/NotificationSettingsDialog.vue
index 69ff2a2d0..a2eb4afd0 100644
--- a/dashboard/src/components/NotificationSettingsDialog.vue
+++ b/dashboard/src/components/NotificationSettingsDialog.vue
@@ -13,6 +13,7 @@ const appUpp = ref(false);
const appDown = ref(false);
const appOutOfMemory = ref(false);
const backupFailed = ref(false);
+const appAutoUpdateFailed = ref(false);
const certificateRenewalFailed = ref(false);
const diskSpace = ref(false);
const cloudronUpdateFailed = ref(false);
@@ -26,6 +27,7 @@ async function onSubmit() {
if (appDown.value) config.push('appDown');
if (appOutOfMemory.value) config.push('appOutOfMemory');
if (backupFailed.value) config.push('backupFailed');
+ if (appAutoUpdateFailed.value) config.push('appAutoUpdateFailed');
if (certificateRenewalFailed.value) config.push('certificateRenewalFailed');
if (diskSpace.value) config.push('diskSpace');
if (cloudronUpdateFailed.value) config.push('cloudronUpdateFailed');
@@ -49,6 +51,7 @@ async function open() {
appDown.value = config.indexOf('appDown') !== -1;
appOutOfMemory.value = config.indexOf('appOutOfMemory') !== -1;
backupFailed.value = config.indexOf('backupFailed') !== -1;
+ appAutoUpdateFailed.value = config.indexOf('appAutoUpdateFailed') !== -1;
certificateRenewalFailed.value = config.indexOf('certificateRenewalFailed') !== -1;
diskSpace.value = config.indexOf('diskSpace') !== -1;
cloudronUpdateFailed.value = config.indexOf('cloudronUpdateFailed') !== -1;
@@ -98,6 +101,11 @@ defineExpose({
+
+ {{ $t('notifications.settings.appAutoUpdateFailed') }}
+
+
+
{{ $t('notifications.settings.certificateRenewalFailed') }}
diff --git a/src/apps.js b/src/apps.js
index f963d9587..d412ced96 100644
--- a/src/apps.js
+++ b/src/apps.js
@@ -1063,8 +1063,9 @@ async function onTaskFinished(error, appId, installationState, taskId, auditSour
case ISTATE_PENDING_UPDATE: {
const fromManifest = success ? task.args[1].updateConfig.manifest : app.manifest;
const toManifest = success ? app.manifest : task.args[1].updateConfig.manifest;
+ const backupError = error?.backupError || false;
- await eventlog.add(eventlog.ACTION_APP_UPDATE_FINISH, auditSource, { app, toManifest, fromManifest, success, errorMessage });
+ await eventlog.add(eventlog.ACTION_APP_UPDATE_FINISH, auditSource, { app, toManifest, fromManifest, success, errorMessage, backupError });
await notifications.unpin(notifications.TYPE_MANUAL_APP_UPDATE_NEEDED, { context: app.id });
break;
}
@@ -2333,7 +2334,7 @@ async function updateApp(app, data, auditSource) {
if (!skipBackup) {
const sites = await backupSites.listByContentForUpdates(app.id);
- if (sites.length === 0) throw new BoxError(BoxError.BAD_STATE, 'App has no backup site for updates');
+ if (sites.length === 0) throw new BoxError(BoxError.BAD_STATE, 'App has no backup site for updates', { backupError: true });
}
const updateConfig = { skipBackup, manifest }; // this will clear appStoreId/versionsUrl when updating from a repo and set it if passed in for update route
diff --git a/src/mail_templates/app_auto_update_failed-text.ejs b/src/mail_templates/app_auto_update_failed-text.ejs
new file mode 100644
index 000000000..352ceaad3
--- /dev/null
+++ b/src/mail_templates/app_auto_update_failed-text.ejs
@@ -0,0 +1,21 @@
+Dear <%= cloudronName %> Admin,
+
+<% if (backupError) { -%>
+The automatic update of <%= appTitle %> at <%= appFqdn %> was skipped because the pre-update backup failed.
+<% } else { -%>
+The automatic update of <%= appTitle %> at <%= appFqdn %> failed.
+<% } -%>
+
+-------------------------------------
+
+<%- message %>
+
+-------------------------------------
+
+The update will be retried on the next automatic update cycle.
+
+Powered by https://cloudron.io
+
+Don't want such mails? Change your notification preferences at <%= notificationsUrl %>
+
+Sent at: <%= new Date().toUTCString() %>
diff --git a/src/mailer.js b/src/mailer.js
index 92ffd7572..56d7d98a2 100644
--- a/src/mailer.js
+++ b/src/mailer.js
@@ -238,6 +238,26 @@ async function oomEvent(mailTo, containerId, app, addon, event) {
await sendMail(mailOptions);
}
+async function appAutoUpdateFailed(mailTo, app, errorMessage, backupError) {
+ assert.strictEqual(typeof mailTo, 'string');
+ assert.strictEqual(typeof app, 'object');
+ assert.strictEqual(typeof errorMessage, 'string');
+ assert.strictEqual(typeof backupError, 'boolean');
+
+ const mailConfig = await getMailConfig();
+
+ const subject = `[${mailConfig.cloudronName}] Automatic update of ${app.fqdn} failed`;
+
+ const mailOptions = {
+ from: mailConfig.notificationFrom,
+ to: mailTo,
+ subject,
+ text: render('app_auto_update_failed-text.ejs', { cloudronName: mailConfig.cloudronName, appFqdn: app.fqdn, appTitle: app.manifest.title, message: errorMessage, backupError, notificationsUrl: mailConfig.notificationsUrl })
+ };
+
+ await sendMail(mailOptions);
+}
+
async function backupFailed(mailTo, errorMessage, logUrl) {
assert.strictEqual(typeof mailTo, 'string');
assert.strictEqual(typeof errorMessage, 'string');
@@ -341,6 +361,7 @@ export default {
passwordReset,
sendInvite,
sendNewLoginLocation,
+ appAutoUpdateFailed,
backupFailed,
certificateRenewalError,
appDown,
diff --git a/src/notifications.js b/src/notifications.js
index ccee1263c..81f5f68f0 100644
--- a/src/notifications.js
+++ b/src/notifications.js
@@ -28,6 +28,7 @@ const TYPE_REBOOT = 'reboot';
const TYPE_UPDATE_UBUNTU = 'ubuntuUpdate';
const TYPE_BOX_UPDATE = 'boxUpdate';
const TYPE_MANUAL_APP_UPDATE_NEEDED = 'manualAppUpdate';
+const TYPE_APP_AUTO_UPDATE_FAILED = 'appAutoUpdateFailed';
const NOTIFICATION_FIELDS = [ 'id', 'eventId', 'type', 'title', 'message', 'creationTime', 'acknowledged', 'context' ];
@@ -242,6 +243,25 @@ async function certificateRenewalError(eventId, fqdn, errorMessage) {
}
}
+async function appAutoUpdateFailed(eventId, app, errorMessage, backupError) {
+ assert.strictEqual(typeof eventId, 'string');
+ assert.strictEqual(typeof app, 'object');
+ assert.strictEqual(typeof errorMessage, 'string');
+ assert.strictEqual(typeof backupError, 'boolean');
+
+ const title = `Automatic update of ${app.manifest.title} at ${app.fqdn} failed`;
+ const message = `The automatic update of the app at https://${app.fqdn} failed: ${errorMessage}`;
+
+ await add(TYPE_APP_AUTO_UPDATE_FAILED, title, message, { eventId });
+
+ const admins = await users.getAdmins();
+ for (const admin of admins) {
+ if (admin.notificationConfig.includes(TYPE_APP_AUTO_UPDATE_FAILED)) {
+ await safe(mailer.appAutoUpdateFailed(admin.email, app, errorMessage, backupError), { debug: log });
+ }
+ }
+}
+
async function backupFailed(eventId, taskId, errorMessage) {
assert.strictEqual(typeof eventId, 'string');
assert.strictEqual(typeof taskId, 'string');
@@ -339,7 +359,7 @@ async function onEvent(eventId, action, source, data) {
case eventlog.ACTION_APP_UPDATE_FINISH:
if (source.username !== AuditSource.CRON.username) return; // updated by user
- if (data.errorMessage) return; // the update indicator will still appear, so no need to notify user
+ if (data.errorMessage) return await appAutoUpdateFailed(eventId, data.app, data.errorMessage, data.backupError);
return await appUpdated(eventId, data.app, data.fromManifest, data.toManifest);
case eventlog.ACTION_CERTIFICATE_RENEWAL:
@@ -391,6 +411,7 @@ export default {
TYPE_UPDATE_UBUNTU,
TYPE_BOX_UPDATE,
TYPE_MANUAL_APP_UPDATE_NEEDED,
+ TYPE_APP_AUTO_UPDATE_FAILED,
TYPE_DOMAIN_CONFIG_CHECK_FAILED,
pin,
unpin,
diff --git a/src/updater.js b/src/updater.js
index 99d9746b4..2f4f8762f 100644
--- a/src/updater.js
+++ b/src/updater.js
@@ -280,12 +280,6 @@ async function autoUpdate(auditSource) {
continue;
}
- const sites = await backupSites.listByContentForUpdates(app.id);
- if (sites.length === 0) {
- log(`autoUpdate: ${app.fqdn} has no backup site for updates. skipping`);
- continue;
- }
-
const data = {
manifest: app.updateInfo.manifest,
force: false
@@ -293,7 +287,11 @@ async function autoUpdate(auditSource) {
log(`autoUpdate: ${app.fqdn} will be automatically updated`);
const [updateError] = await safe(apps.updateApp(app, data, auditSource));
- if (updateError) log(`autoUpdate: error autoupdating ${app.fqdn}: ${updateError.message}`);
+ if (updateError) {
+ log(`autoUpdate: error autoupdating ${app.fqdn}: ${updateError.message}`);
+ await safe(eventlog.add(eventlog.ACTION_APP_UPDATE_FINISH, auditSource, { app, toManifest: data.manifest, fromManifest: app.manifest, success: false,
+ errorMessage: updateError.message, backupError: !!updateError.backupError }), { debug: log });
+ }
}
}
@@ -343,9 +341,11 @@ async function raiseNotifications() {
const result = await apps.list();
for (const app of result) {
+ if (!app.updateInfo) continue;
+
// currently, we do not raise notifications when auto-update is disabled. separate notifications appears spammy when having many apps
// in the future, we can maybe aggregate?
- if (app.updateInfo && !app.updateInfo.isAutoUpdatable) {
+ if (!app.updateInfo.isAutoUpdatable) {
log(`autoUpdate: ${app.fqdn} cannot be autoupdated. skipping`);
await notifications.pin(notifications.TYPE_MANUAL_APP_UPDATE_NEEDED, `${app.manifest.title} at ${app.fqdn} requires manual update to version ${app.updateInfo.manifest.version}`,
`Changelog:\n${app.updateInfo.manifest.changelog}\n`, { context: app.id });
diff --git a/src/users.js b/src/users.js
index c0ff3bffd..fb254a7e6 100644
--- a/src/users.js
+++ b/src/users.js
@@ -837,7 +837,7 @@ async function createOwner(email, username, password, displayName, auditSource)
const activated = await isActivated();
if (activated) throw new BoxError(BoxError.ALREADY_EXISTS, 'Cloudron already activated');
- const notificationConfig = [notifications.TYPE_BACKUP_FAILED, notifications.TYPE_CERTIFICATE_RENEWAL_FAILED, notifications.TYPE_MANUAL_APP_UPDATE_NEEDED, notifications.TYPE_APP_DOWN, notifications.TYPE_CLOUDRON_UPDATE_FAILED ];
+ const notificationConfig = [notifications.TYPE_BACKUP_FAILED, notifications.TYPE_CERTIFICATE_RENEWAL_FAILED, notifications.TYPE_MANUAL_APP_UPDATE_NEEDED, notifications.TYPE_APP_DOWN, notifications.TYPE_CLOUDRON_UPDATE_FAILED, notifications.TYPE_APP_AUTO_UPDATE_FAILED ];
return await add(email, { username, password, fallbackEmail: '', displayName, role: ROLE_OWNER, notificationConfig }, auditSource);
}