notifications: only send backup failure email on 3 consecutive fails

src/auditsource.js

@@ -6,10 +6,10 @@ exports = module.exports = {
     EXTERNAL_LDAP_TASK: { userId: null, username: 'externalldap' },
     EXTERNAL_LDAP_AUTO_CREATE: { userId: null, username: 'externalldap' },
 
-    fromRequest: fromRequest
+    fromRequest
 };
 
 function fromRequest(req) {
-    var ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress || null;
-    return { ip: ip, username: req.user ? req.user.username : null, userId: req.user ? req.user.id : null };
+    const ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress || null;
+    return { ip, username: req.user ? req.user.username : null, userId: req.user ? req.user.id : null };
 }

src/notifications.js

@@ -194,6 +194,17 @@ async function backupFailed(eventId, taskId, errorMessage) {
     assert.strictEqual(typeof taskId, 'string');
     assert.strictEqual(typeof errorMessage, 'string');
 
+    await add(eventId, 'Backup failed', `Backup failed: ${errorMessage}. Logs are available [here](/logs.html?taskId=${taskId}).`);
+
+    // only send mail if the past 3 automated backups failed
+    const backupEvents = await eventlog.getAllPaged(eventlog.ACTION_BACKUP_FINISH, null /* search */, 1, 20);
+    let count = 0;
+    for (const event of backupEvents) {
+        if (!event.data.errorMessage) return; // successful backup (manual or cron)
+        if (event.source.username === auditSource.CRON.username && ++count === 3) break; // last 3 consecutive crons have failed
+    }
+    if (count !== 3) return; // less than 3 failures
+
     const getSuperAdmins = util.callbackify(users.getSuperAdmins);
 
     const superAdmins = await getSuperAdmins();
@@ -201,8 +212,6 @@ async function backupFailed(eventId, taskId, errorMessage) {
     for (const superAdmin of superAdmins) {
         mailer.backupFailed(superAdmin.email, errorMessage, `${settings.dashboardOrigin()}/logs.html?taskId=${taskId}`);
     }
-
-    await add(eventId, 'Backup failed', `Backup failed: ${errorMessage}. Logs are available [here](/logs.html?taskId=${taskId}).`);
 }
 
 // id is unused but nice to search code