retry startup tasks on database error

https://forum.cloudron.io/topic/5909/cloudron-7-0-1-gitlab-stuck-after-update

src/cloudron.js

@@ -58,7 +58,7 @@ const apps = require('./apps.js'),
 const REBOOT_CMD = path.join(__dirname, 'scripts/reboot.sh');
 
 async function initialize() {
-    safe(runStartupTasks(), { debug });
+    safe(runStartupTasks(), { debug }); // background
 
     await notifyUpdate();
 }
@@ -152,8 +152,16 @@ async function runStartupTasks() {
 
     // we used to run tasks in parallel but simultaneous nginx reloads was causing issues
     for (let i = 0; i < tasks.length; i++) {
-        const [error] = await safe(tasks[i]());
-        if (error) debug(`Startup task at index ${i} failed: ${error.message}`);
+        for (let attempt = 0; attempt < 3; attempt++) {
+            const [error] = await safe(tasks[i]());
+            if (!error) break; // task succeeded
+            debug(`Startup task at index ${i} failed (attempt ${attempt}): ${error.message}`);
+            // for some reason, mysql arbitrary restarts making startup tasks fail. this makes the box update stuck
+            const retry = error.reason === BoxError.DATABASE_ERROR && error.code === 'PROTOCOL_CONNECTION_LOST';
+            if (!retry) break;
+            debug(`Will retry task at index ${i}`);
+            await delay(3000);
+        }
     }
 }