diff --git a/package-lock.json b/package-lock.json index 0d091d61e..bef14f724 100644 --- a/package-lock.json +++ b/package-lock.json @@ -60,6 +60,7 @@ "commander": "^14.0.0", "easy-table": "^1.2.0", "eslint": "^9.31.0", + "eventsource": "^4.0.0", "expect.js": "*", "mocha": "^11.7.1", "nock": "^14.0.5", @@ -4519,6 +4520,29 @@ "node": ">=6" } }, + "node_modules/eventsource": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/eventsource/-/eventsource-4.0.0.tgz", + "integrity": "sha512-fvIkb9qZzdMxgZrEQDyll+9oJsyaVvY92I2Re+qK0qEJ+w5s0X3dtz+M0VAPOjP1gtU3iqWyjQ0G3nvd5CLZ2g==", + "dev": true, + "license": "MIT", + "dependencies": { + "eventsource-parser": "^3.0.1" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/eventsource-parser": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.0.3.tgz", + "integrity": "sha512-nVpZkTMM9rF6AQ9gPJpFsNAMt48wIzB5TQgiTLdHiuO8XEDhUgZEhqKlZWXbIzo9VmJ/HvysHqEaVeD5v9TPvA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=20.0.0" + } + }, "node_modules/expect.js": { "version": "0.3.1", "dev": true diff --git a/package.json b/package.json index 64a7211f5..2a9306a58 100644 --- a/package.json +++ b/package.json @@ -64,6 +64,7 @@ "commander": "^14.0.0", "easy-table": "^1.2.0", "eslint": "^9.31.0", + "eventsource": "^4.0.0", "expect.js": "*", "mocha": "^11.7.1", "nock": "^14.0.5", diff --git a/src/asynctask.js b/src/asynctask.js index 12239c421..8c60ccaa7 100644 --- a/src/asynctask.js +++ b/src/asynctask.js @@ -6,12 +6,12 @@ const debug = require('debug')('box:asynctask'), // this runs in-process class AsyncTask extends EventEmitter { - #name; + name; #abortController; constructor(name) { super(); - this.#name = name; + this.name = name; this.#abortController = new AbortController(); } @@ -20,22 +20,19 @@ class AsyncTask extends EventEmitter { } async start() { - debug(`start: ${this.#name} started`); + debug(`start: ${this.name} started`); const [error] = await safe(this._run(this.#abortController.signal)); // background - debug(`start: ${this.#name} done`, error); - this.done(error); + debug(`start: ${this.name} finished`); + this.emit('done', { errorMessage: error?.message || '' }); + this.#abortController = null; } stop() { - debug(`stop: ${this.#name} stopped`); + if (this.#abortController === null) return; // already finished + debug(`stop: ${this.name} . sending abort signal`); this.#abortController.abort(); } - done(error) { - debug(`done: ${this.#name} finished`); - this.emit('done', { errorMessage: error?.message || '' }); - } - emitProgress(percent, message) { this.emit('data', 'progress', { percent, message }); } diff --git a/src/docker.js b/src/docker.js index 6965539c2..386e32271 100644 --- a/src/docker.js +++ b/src/docker.js @@ -679,8 +679,10 @@ async function info() { return result; } -async function df() { - const [error, result] = await safe(gConnection.df()); +async function df(options) { + assert.strictEqual(typeof options, 'object'); + + const [error, result] = await safe(gConnection.df(options)); if (error) throw new BoxError(BoxError.DOCKER_ERROR, `Error connecting to docker: ${error.message}`); return result; } diff --git a/src/routes/test/common.js b/src/routes/test/common.js index 6e29807d8..c18e9ae06 100644 --- a/src/routes/test/common.js +++ b/src/routes/test/common.js @@ -6,12 +6,10 @@ const apps = require('../../apps.js'), constants = require('../../constants.js'), database = require('../../database.js'), expect = require('expect.js'), - fs = require('fs'), mailer = require('../../mailer.js'), nock = require('nock'), oidcClients = require('../../oidcclients.js'), oidcServer = require('../../oidcserver.js'), - safe = require('safetydance'), server = require('../../server.js'), settings = require('../../settings.js'), superagent = require('@cloudron/superagent'), @@ -54,6 +52,7 @@ exports = module.exports = { clearMailQueue, checkMails, waitForTask, + waitForAsyncTask, owner: { id: null, @@ -179,6 +178,7 @@ async function cleanup() { debug('Cleaning up'); await server.stop(); await oidcServer.stop(); + if (!nock.isActive()) nock.activate(); debug('Cleaned up'); } @@ -207,3 +207,25 @@ async function waitForTask(taskId) { } throw new Error(`Task ${taskId} never finished`); } + +async function waitForAsyncTask(es) { + return new Promise((resolve, reject) => { + const messages = []; + es.addEventListener('message', function (message) { + debug(`waitForAsyncTask: ${message.data}`); + messages.push(JSON.parse(message.data)); + if (messages[messages.length-1].type === 'done') { + debug('waitForAsyncTask: finished'); + es.close(); + resolve(messages); + } + }); + es.addEventListener('error', function (error) { + debug('waitForAsyncTask: errored', error); + es.close(); + const e = new Error(error.message); + e.code = error.code; + reject(e); + }); + }); +} diff --git a/src/routes/test/system-test.js b/src/routes/test/system-test.js index c74bcfbed..6ed4a5c34 100644 --- a/src/routes/test/system-test.js +++ b/src/routes/test/system-test.js @@ -7,18 +7,26 @@ const constants = require('../../constants.js'), common = require('./common.js'), + { EventSource } = require('eventsource'), expect = require('expect.js'), fs = require('fs'), http = require('http'), + nock = require('nock'), os = require('os'), paths = require('../../paths.js'), + safe = require('safetydance'), superagent = require('@cloudron/superagent'); describe('System', function () { - const { setup, cleanup, serverUrl, owner, user, waitForTask } = common; + const { setup, cleanup, serverUrl, owner, user, waitForAsyncTask } = common; - before(setup); - after(cleanup); + before(async function () { + await setup(); + if (nock.isActive()) nock.restore(); // the docker df call does not go through otherwise + }); + after(async function () { + await cleanup(); + }); describe('cpus', function () { it('succeeds', async function () { @@ -127,41 +135,31 @@ describe('System', function () { }); }); - describe('disk usage', function () { - it('get succeeds with no cache', async function () { - const response = await superagent.get(`${serverUrl}/api/v1/system/disk_usage`) + describe('filesystem', function () { + let rootFs; + + it('get filesystems', async function () { + const response = await superagent.get(`${serverUrl}/api/v1/system/filesystems`) .query({ access_token: owner.token }); expect(response.status).to.equal(200); - expect(response.body).to.eql({ usage: null }); + + rootFs = Object.values(response.body.filesystems).find(v => v.mountpoint === '/'); + expect(rootFs.filesystem).to.be.ok(); }); - it('update the cache', async function () { - const response = await superagent.post(`${serverUrl}/api/v1/system/disk_usage`) - .query({ access_token: owner.token }) - .send({}); - - expect(response.status).to.equal(201); - expect(response.body.taskId).to.be.ok(); - await waitForTask(response.body.taskId); + it('fails without query param', async function () { + const es = new EventSource(`${serverUrl}/api/v1/system/filesystem_usage?access_token=${owner.token}`); + const [error] = await safe(waitForAsyncTask(es)); + expect(error.code).to.be(400); }); - it('get succeeds with cache', async function () { - const response = await superagent.get(`${serverUrl}/api/v1/system/disk_usage`) - .query({ access_token: owner.token }); - - expect(response.status).to.equal(200); - expect(response.body.usage.ts).to.be.a('number'); - - const filesystems = Object.keys(response.body.usage.filesystems); - let dockerUsage = null; - for (const fs of filesystems) { - for (const content of response.body.usage.filesystems[fs].contents) { - if (content.id === 'docker') dockerUsage = content; - } - } - expect(dockerUsage).to.be.ok(); - expect(dockerUsage.usage).to.be.a('number'); + it('succceeds with query param', async function () { + const es = new EventSource(`${serverUrl}/api/v1/system/filesystem_usage?access_token=${owner.token}&filesystem=${rootFs.filesystem}`); + const messages = await waitForAsyncTask(es); + expect(messages.find(m => m.type === 'progress')).to.be.ok(); + expect(messages.find(m => m.type === 'data')).to.be.ok(); + expect(messages.find(m => m.type === 'done')).to.be.ok(); }); }); diff --git a/src/server.js b/src/server.js index 7f1939e2f..64efadf86 100644 --- a/src/server.js +++ b/src/server.js @@ -121,7 +121,7 @@ async function initializeExpressSync() { router.get ('/api/v1/system/metricstream', token, authorizeAdmin, routes.system.getMetricStream); router.get ('/api/v1/system/block_devices', token, authorizeAdmin, routes.system.getBlockDevices); router.get ('/api/v1/system/filesystems', token, authorizeAdmin, routes.system.getFilesystems); - router.post('/api/v1/system/filesystem_usage', token, authorizeAdmin, routes.system.getFilesystemUsage); + router.get ('/api/v1/system/filesystem_usage', token, authorizeAdmin, routes.system.getFilesystemUsage); router.get ('/api/v1/system/logs/:unit', token, authorizeAdmin, routes.system.getLogs); router.get ('/api/v1/system/logstream/:unit', token, authorizeAdmin, routes.system.getLogStream); // app operators require cpu and memory info for the Resources UI diff --git a/src/system.js b/src/system.js index 580ee62bb..dc7010bb6 100644 --- a/src/system.js +++ b/src/system.js @@ -240,10 +240,10 @@ class FilesystemUsageTask extends AsyncTask { this.emitData({ speed: -1 }); } - const dockerDf = await docker.df(); + const dockerDf = await docker.df({ abortSignal: signal }); for (const content of contents) { - percent += (100/contents.length); + percent += (90/contents.length+1); if (signal.aborted) return; this.emitProgress(percent,`Checking du of ${content.id} ${content.path}`); diff --git a/src/test/docker-test.js b/src/test/docker-test.js index 9c33a1ec5..1b8503f3d 100644 --- a/src/test/docker-test.js +++ b/src/test/docker-test.js @@ -22,7 +22,7 @@ describe('docker', function () { after(cleanup); it('can df', async function () { - const output = await docker.df(); + const output = await docker.df({}); expect(output).to.be.ok(); });