notifications: only send backup failure email on 3 consecutive fails

This commit is contained in:
Girish Ramakrishnan
2021-06-23 16:50:19 -07:00
parent 6da7218d34
commit 2b13593630
2 changed files with 14 additions and 5 deletions

View File

@@ -194,6 +194,17 @@ async function backupFailed(eventId, taskId, errorMessage) {
assert.strictEqual(typeof taskId, 'string');
assert.strictEqual(typeof errorMessage, 'string');
await add(eventId, 'Backup failed', `Backup failed: ${errorMessage}. Logs are available [here](/logs.html?taskId=${taskId}).`);
// only send mail if the past 3 automated backups failed
const backupEvents = await eventlog.getAllPaged(eventlog.ACTION_BACKUP_FINISH, null /* search */, 1, 20);
let count = 0;
for (const event of backupEvents) {
if (!event.data.errorMessage) return; // successful backup (manual or cron)
if (event.source.username === auditSource.CRON.username && ++count === 3) break; // last 3 consecutive crons have failed
}
if (count !== 3) return; // less than 3 failures
const getSuperAdmins = util.callbackify(users.getSuperAdmins);
const superAdmins = await getSuperAdmins();
@@ -201,8 +212,6 @@ async function backupFailed(eventId, taskId, errorMessage) {
for (const superAdmin of superAdmins) {
mailer.backupFailed(superAdmin.email, errorMessage, `${settings.dashboardOrigin()}/logs.html?taskId=${taskId}`);
}
await add(eventId, 'Backup failed', `Backup failed: ${errorMessage}. Logs are available [here](/logs.html?taskId=${taskId}).`);
}
// id is unused but nice to search code