01d0c738bc
mostly we want trace() and log(). trace() can be enabled whenever we want by flipping a flag and restarting box
311 lines
10 KiB
JavaScript
311 lines
10 KiB
JavaScript
import appHealthMonitor from './apphealthmonitor.js';
|
|
import assert from 'node:assert';
|
|
import appstore from './appstore.js';
|
|
import AuditSource from './auditsource.js';
|
|
import backupSites from './backupsites.js';
|
|
import cloudron from './cloudron.js';
|
|
import constants from './constants.js';
|
|
import { CronJob } from 'cron';
|
|
import logger from './logger.js';
|
|
import domains from './domains.js';
|
|
import dyndns from './dyndns.js';
|
|
import externalLdap from './externalldap.js';
|
|
import eventlog from './eventlog.js';
|
|
import janitor from './janitor.js';
|
|
import mail from './mail.js';
|
|
import metrics from './metrics.js';
|
|
import network from './network.js';
|
|
import oidcServer from './oidcserver.js';
|
|
import paths from './paths.js';
|
|
import reverseProxy from './reverseproxy.js';
|
|
import safe from 'safetydance';
|
|
import scheduler from './scheduler.js';
|
|
import system from './system.js';
|
|
import updater from './updater.js';
|
|
import util from 'node:util';
|
|
|
|
const { log, trace } = logger('cron');
|
|
|
|
// IMPORTANT: These patterns are together because they spin tasks which acquire a lock
|
|
// If the patterns overlap all the time, then the task may not ever get a chance to run!
|
|
// If you change this change dashboard patterns in settings.html
|
|
const DEFAULT_CLEANUP_BACKUPS_PATTERN = '00 30 1,3,5,23 * * *',
|
|
DEFAULT_AUTOUPDATE_PATTERN = '00 00 1,3,5,23 * * *';
|
|
|
|
|
|
const gJobs = {
|
|
autoUpdater: null,
|
|
backups: new Map(),
|
|
updateChecker: null,
|
|
systemChecks: null,
|
|
mailStatusCheck: null,
|
|
diskSpaceChecker: null,
|
|
certificateRenew: null,
|
|
cleanupBackups: null,
|
|
cleanupEventlog: null,
|
|
cleanupTokens: null,
|
|
cleanupOidc: null,
|
|
dockerVolumeCleaner: null,
|
|
dynamicDns: null,
|
|
schedulerSync: null,
|
|
appHealthMonitor: null,
|
|
externalLdapSyncer: null,
|
|
checkDomainConfigs: null,
|
|
collectStats: null,
|
|
subscriptionChecker: null,
|
|
};
|
|
|
|
// cron format
|
|
// Seconds: 0-59
|
|
// Minutes: 0-59
|
|
// Hours: 0-23
|
|
// Day of Month: 1-31
|
|
// Months: 0-11
|
|
// Day of Week: 0-6
|
|
|
|
function getCronSeed() {
|
|
let hour = null;
|
|
let minute = null;
|
|
|
|
const seedData = safe.fs.readFileSync(paths.CRON_SEED_FILE, 'utf8') || '';
|
|
const parts = seedData.split(':');
|
|
if (parts.length === 2) {
|
|
hour = parseInt(parts[0]) || null;
|
|
minute = parseInt(parts[1]) || null;
|
|
}
|
|
|
|
if ((hour == null || hour < 0 || hour > 23) || (minute == null || minute < 0 || minute > 60)) {
|
|
hour = Math.floor(24 * Math.random());
|
|
minute = Math.floor(60 * Math.random());
|
|
|
|
log(`getCronSeed: writing new cron seed file with ${hour}:${minute} to ${paths.CRON_SEED_FILE}`);
|
|
|
|
safe.fs.writeFileSync(paths.CRON_SEED_FILE, `${hour}:${minute}`);
|
|
}
|
|
|
|
return { hour, minute };
|
|
}
|
|
|
|
async function handleBackupScheduleChanged(site) {
|
|
assert.strictEqual(typeof site, 'object');
|
|
|
|
const tz = await cloudron.getTimeZone();
|
|
|
|
log(`handleBackupScheduleChanged: schedule ${site.schedule} (${tz})`);
|
|
|
|
if (gJobs.backups.has(site.id)) gJobs.backups.get(site.id).stop();
|
|
gJobs.backups.delete(site.id);
|
|
|
|
if (site.schedule === constants.CRON_PATTERN_NEVER) return;
|
|
|
|
const job = CronJob.from({
|
|
cronTime: site.schedule,
|
|
onTick: async () => {
|
|
const t = await backupSites.get(site.id);
|
|
if (!t) return;
|
|
await safe(backupSites.startBackupTask(t, AuditSource.CRON), { debug: log });
|
|
},
|
|
start: true,
|
|
timeZone: tz
|
|
});
|
|
gJobs.backups.set(site.id, job);
|
|
}
|
|
|
|
async function handleAutoupdatePatternChanged(pattern) {
|
|
assert.strictEqual(typeof pattern, 'string');
|
|
|
|
const tz = await cloudron.getTimeZone();
|
|
|
|
log(`autoupdatePatternChanged: pattern - ${pattern} (${tz})`);
|
|
|
|
if (gJobs.autoUpdater) gJobs.autoUpdater.stop();
|
|
gJobs.autoUpdater = null;
|
|
|
|
if (pattern === constants.CRON_PATTERN_NEVER) return;
|
|
|
|
gJobs.autoUpdater = CronJob.from({
|
|
cronTime: pattern,
|
|
onTick: async () => await safe(updater.autoUpdate(AuditSource.CRON), { debug: log }),
|
|
start: true,
|
|
timeZone: tz
|
|
});
|
|
}
|
|
|
|
function handleDynamicDnsChanged(enabled) {
|
|
assert.strictEqual(typeof enabled, 'boolean');
|
|
|
|
log('Dynamic DNS setting changed to %s', enabled);
|
|
|
|
if (gJobs.dynamicDns) gJobs.dynamicDns.stop();
|
|
gJobs.dynamicDns = null;
|
|
|
|
if (!enabled) return;
|
|
|
|
gJobs.dynamicDns = CronJob.from({
|
|
// until we can be smarter about actual IP changes, lets ensure it every 10minutes
|
|
cronTime: '00 */10 * * * *',
|
|
onTick: async () => { await safe(dyndns.refreshDns(AuditSource.CRON), { debug: log }); },
|
|
start: true
|
|
});
|
|
}
|
|
|
|
async function handleExternalLdapChanged(config) {
|
|
assert.strictEqual(typeof config, 'object');
|
|
|
|
if (gJobs.externalLdapSyncer) gJobs.externalLdapSyncer.stop();
|
|
gJobs.externalLdapSyncer = null;
|
|
|
|
if (config.provider === 'noop') return;
|
|
|
|
gJobs.externalLdapSyncer = CronJob.from({
|
|
cronTime: '00 00 */4 * * *', // every 4 hours
|
|
onTick: async () => await safe(externalLdap.startSyncer(AuditSource.CRON), { debug: log }),
|
|
start: true
|
|
});
|
|
}
|
|
|
|
async function startJobs() {
|
|
const { hour, minute } = getCronSeed();
|
|
|
|
log(`startJobs: starting cron jobs with hour ${hour} and minute ${minute}`);
|
|
|
|
gJobs.systemChecks = CronJob.from({
|
|
cronTime: `00 ${minute} 2 * * *`, // once a day. if you change this interval, change the notification messages with correct duration
|
|
onTick: async () => await safe(system.runSystemChecks(), { debug: log }),
|
|
start: true
|
|
});
|
|
|
|
gJobs.mailStatusCheck = CronJob.from({
|
|
cronTime: `00 ${minute} 2 * * *`, // once a day. if you change this interval, change the notification messages with correct duration
|
|
onTick: async () => await safe(mail.checkStatus(), { debug: log }),
|
|
start: true
|
|
});
|
|
|
|
gJobs.diskSpaceChecker = CronJob.from({
|
|
cronTime: '00 30 * * * *', // every 30 minutes. if you change this interval, change the notification messages with correct duration
|
|
onTick: async () => await safe(system.checkDiskSpace(), { debug: log }),
|
|
start: true
|
|
});
|
|
|
|
// this is run separately from the update itself so that the user can disable automatic updates but can still get a notification
|
|
gJobs.updateCheckerJob = CronJob.from({
|
|
cronTime: `00 ${minute} 1,5,9,13,17,21,23 * * *`,
|
|
onTick: async () => await safe(updater.checkForUpdates({ stableOnly: true }), { debug: log }),
|
|
start: true
|
|
});
|
|
|
|
gJobs.cleanupTokens = CronJob.from({
|
|
cronTime: '00 */30 * * * *', // every 30 minutes
|
|
onTick: async () => await safe(janitor.cleanupTokens(), { debug: log }),
|
|
start: true
|
|
});
|
|
|
|
gJobs.cleanupOidc = CronJob.from({
|
|
cronTime: '00 10 * * * *', // every hour ten minutes past
|
|
onTick: async () => await safe(oidcServer.cleanupExpired(), { debug: log }),
|
|
start: true
|
|
});
|
|
|
|
gJobs.cleanupBackups = CronJob.from({
|
|
cronTime: DEFAULT_CLEANUP_BACKUPS_PATTERN,
|
|
onTick: async () => {
|
|
for (const backupSite of await backupSites.list()) {
|
|
await safe(backupSites.startCleanupTask(backupSite, AuditSource.CRON), { debug: log });
|
|
}
|
|
},
|
|
start: true
|
|
});
|
|
|
|
gJobs.cleanupEventlog = CronJob.from({
|
|
cronTime: '00 */30 * * * *', // every 30 minutes
|
|
onTick: async () => await safe(eventlog.cleanup({ creationTime: new Date(Date.now() - 90 * 60 * 24 * 60 * 1000) }), { debug: log }), // 90 days ago
|
|
start: true
|
|
});
|
|
|
|
gJobs.dockerVolumeCleaner = CronJob.from({
|
|
cronTime: '00 00 */12 * * *', // every 12 hours
|
|
onTick: async () => await safe(janitor.cleanupDockerVolumes(), { debug: log }),
|
|
start: true
|
|
});
|
|
|
|
gJobs.schedulerSync = CronJob.from({
|
|
cronTime: constants.TEST ? '*/10 * * * * *' : '00 */1 * * * *', // every minute
|
|
onTick: async () => await safe(scheduler.sync(), { debug: log }),
|
|
start: true
|
|
});
|
|
|
|
// randomized per Cloudron based on hourlySeed
|
|
gJobs.certificateRenew = CronJob.from({
|
|
cronTime: `00 10 ${hour} * * *`,
|
|
onTick: async () => await safe(reverseProxy.startRenewCerts({}, AuditSource.CRON), { debug: log }),
|
|
start: true
|
|
});
|
|
|
|
gJobs.checkDomainConfigs = CronJob.from({
|
|
cronTime: `00 ${minute} 5 * * *`, // once a day
|
|
onTick: async () => await safe(domains.checkConfigs(AuditSource.CRON), { debug: log }),
|
|
start: true
|
|
});
|
|
|
|
gJobs.appHealthMonitor = CronJob.from({
|
|
cronTime: '*/10 * * * * *', // every 10 seconds
|
|
onTick: async () => await safe(appHealthMonitor.run(10), { debug: log }), // 10 is the max run time
|
|
start: true
|
|
});
|
|
|
|
gJobs.collectStats = CronJob.from({
|
|
cronTime: '*/20 * * * * *', // every 20 seconds. if you change this, change carbon config
|
|
onTick: async () => await safe(metrics.sendToGraphite(), { debug: log }),
|
|
start: true
|
|
});
|
|
|
|
gJobs.subscriptionChecker = CronJob.from({
|
|
cronTime: `00 ${minute} ${hour} * * *`, // once a day based on seed to randomize
|
|
onTick: async () => await safe(appstore.checkSubscription(), { debug: log }),
|
|
start: true
|
|
});
|
|
|
|
for (const backupSite of await backupSites.list()) {
|
|
await handleBackupScheduleChanged(backupSite);
|
|
}
|
|
await handleAutoupdatePatternChanged(await updater.getAutoupdatePattern());
|
|
await handleDynamicDnsChanged(await network.getDynamicDns());
|
|
await handleExternalLdapChanged(await externalLdap.getConfig());
|
|
}
|
|
|
|
async function stopJobs() {
|
|
for (const jobName in gJobs) {
|
|
if (!gJobs[jobName]) continue;
|
|
if (util.types.isMap(gJobs[jobName])) {
|
|
gJobs[jobName].forEach((job) => job.stop());
|
|
gJobs[jobName] = new Map();
|
|
} else {
|
|
gJobs[jobName].stop();
|
|
gJobs[jobName] = null;
|
|
}
|
|
}
|
|
}
|
|
|
|
async function handleTimeZoneChanged(tz) {
|
|
assert.strictEqual(typeof tz, 'string');
|
|
|
|
log('handleTimeZoneChanged: recreating all jobs');
|
|
await stopJobs();
|
|
await scheduler.deleteJobs(); // have to re-create with new tz
|
|
await startJobs();
|
|
}
|
|
|
|
export default {
|
|
startJobs,
|
|
|
|
stopJobs,
|
|
|
|
handleBackupScheduleChanged,
|
|
handleTimeZoneChanged,
|
|
handleAutoupdatePatternChanged,
|
|
handleDynamicDnsChanged,
|
|
handleExternalLdapChanged,
|
|
|
|
DEFAULT_AUTOUPDATE_PATTERN,
|
|
};
|