diff --git a/CHANGES b/CHANGES index 6b241ce95..ad74fe7a1 100644 --- a/CHANGES +++ b/CHANGES @@ -2373,6 +2373,7 @@ * volumes: ensure we don't crash if mount status is unexpected * backups: set default backup memory limit to 800 * users: allow admins to specify password recovery email +* retry startup tasks on database error [7.0.3] * support: fix remoe support not working for 'root' user diff --git a/src/cloudron.js b/src/cloudron.js index 9ba6f4542..0e835a69e 100644 --- a/src/cloudron.js +++ b/src/cloudron.js @@ -58,7 +58,7 @@ const apps = require('./apps.js'), const REBOOT_CMD = path.join(__dirname, 'scripts/reboot.sh'); async function initialize() { - safe(runStartupTasks(), { debug }); + safe(runStartupTasks(), { debug }); // background await notifyUpdate(); } @@ -152,8 +152,16 @@ async function runStartupTasks() { // we used to run tasks in parallel but simultaneous nginx reloads was causing issues for (let i = 0; i < tasks.length; i++) { - const [error] = await safe(tasks[i]()); - if (error) debug(`Startup task at index ${i} failed: ${error.message}`); + for (let attempt = 0; attempt < 3; attempt++) { + const [error] = await safe(tasks[i]()); + if (!error) break; // task succeeded + debug(`Startup task at index ${i} failed (attempt ${attempt}): ${error.message}`); + // for some reason, mysql arbitrary restarts making startup tasks fail. this makes the box update stuck + const retry = error.reason === BoxError.DATABASE_ERROR && error.code === 'PROTOCOL_CONNECTION_LOST'; + if (!retry) break; + debug(`Will retry task at index ${i}`); + await delay(3000); + } } }