tasks: distinguish runtime crash vs task error in worker

src/apps.js

@@ -1271,8 +1271,8 @@ async function scheduleTask(appId, installationState, taskId, auditSource) {
     const options = { timeout: 20 * 60 * 60 * 1000 /* 20 hours */, nice: 15, memoryLimit };
 
     appTaskManager.scheduleTask(appId, taskId, options, async function (error) {
-        debug(`scheduleTask: task ${taskId} of ${appId} completed`);
-        if (error && (error.code === tasks.ECRASHED || error.code === tasks.ESTOPPED)) { // if task crashed, update the error
+        debug(`scheduleTask: task ${taskId} of ${appId} completed. error: %o`, error);
+        if (error?.code === tasks.ECRASHED || error?.code === tasks.ESTOPPED) { // if task crashed, update the error
             debug(`Apptask crashed/stopped: ${error.message}`);
             const boxError = new BoxError(BoxError.TASK_ERROR, error.message);
             boxError.details.crashed = error.code === tasks.ECRASHED;

src/tasks.js

@@ -83,11 +83,11 @@ function postProcess(task) {
     // result.pending - task is scheduled to run at some point
     // result.completed - task finished and exit/crash was cleanly collected. internal flag.
     task.running = !!gTasks[task.id]; // running means actively running
-    task.active = task.running || task.pending; // active mean task is 'done' or not. at this point, clients can stop polling this task.
+    task.active = task.running || task.pending; // active mean task is 'done'. at this point, clients can stop polling this task.
     task.success = task.completed && !task.error; // if task has completed without an error
 
-    // the error in db will be empty if we didn't get a chance to handle task exit
-    if (!task.active && !task.completed && !task.error) {
+    // the error in db will be empty if task is done but the completed flag is not set
+    if (!task.active && !task.completed) {
         task.error = { message: 'Task was stopped because the server restarted or crashed', code: exports.ECRASHED };
     }
 

src/taskworker.js

@@ -5,6 +5,7 @@
 const apptask = require('./apptask.js'),
     backupCleaner = require('./backupcleaner.js'),
     backuptask = require('./backuptask.js'),
+    BoxError = require('./boxerror.js'),
     dashboard = require('./dashboard.js'),
     database = require('./database.js'),
     dns = require('./dns.js'),
@@ -73,6 +74,15 @@ function exitSync(status) {
     process.exit(status.code);
 }
 
+function toTaskError(runError) {
+    if (runError instanceof BoxError) return runError.toPlainObject();
+    return {
+        message: `Task crashed. ${runError.message}`,
+        stack: runError.stack,
+        code: tasks.ECRASHED
+    };
+}
+
 // Main process starts here
 const startTime = new Date();
 
@@ -107,22 +117,18 @@ async function main() {
         await safe(tasks.update(taskId, progress), { debug });
     }
 
-    try {
-        debug(`Running task of type ${task.type}`);
-        const [runError, result] = await safe(TASKS[task.type.replace(/_.*/,'')].apply(null, task.args.concat(progressCallback)));
-        const progress = {
-            result: result || null,
-            error: runError ? JSON.parse(JSON.stringify(runError, Object.getOwnPropertyNames(runError))) : null,
-            percent: 100
-        };
+    debug(`Running task of type ${task.type}`);
+    const [runError, result] = await safe(TASKS[task.type.replace(/_.*/,'')].apply(null, task.args.concat(progressCallback)));
+    const progress = {
+        result: result || null,
+        error: runError ? toTaskError(runError) : null,
+        percent: 100
+    };
 
-        debug(`Task took ${(new Date() - startTime)/1000} seconds`);
+    debug(`Task took ${(new Date() - startTime)/1000} seconds`);
 
-        await safe(tasks.setCompleted(taskId, progress));
-        exitSync({ error: runError, code: 0 }); // code itself ran fine, but resulted in some error. so exit with success
-    } catch (error) {
-        exitSync({ error, code: 50 }); // do not call setCompleted() intentionally. the task code must be resilient enough to handle it
-    }
+    await safe(tasks.setCompleted(taskId, progress), { debug });
+    exitSync({ error: runError, code: runError instanceof BoxError ? 0 : 50 }); // handled error vs run time crash
 }
 
 main();