jjkiers/cloudron-box
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

superagent: rewrite using native node request

Browse Source 
the learning is that fetch() is really meant to be a browser side
XMLHttpRequest replacement. It's complicated to do things like
setting user agent, custom headers like Host, disabling tls validation etc.
This commit is contained in:
Girish Ramakrishnan
2025-02-18 15:23:32 +01:00
parent a019227ddc
commit d75e95a23d
4 changed files with 103 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
package-lock.json generated
View File

@@ -50,7 +50,6 @@
"tar-stream": "^3.1.7",
"tldjs": "^2.3.1",
"ua-parser-js": "^2.0.2",
"undici": "^7.3.0",
"uuid": "^11.0.5",
"validator": "^13.12.0",
"ws": "^8.18.0",
@@ -8129,15 +8128,6 @@
"node": ">= 0.8"
}
},
"node_modules/undici": {
"version": "7.3.0",
"resolved": "https://registry.npmjs.org/undici/-/undici-7.3.0.tgz",
"integrity": "sha512-Qy96NND4Dou5jKoSJ2gm8ax8AJM/Ey9o9mz7KN1bb9GP+G0l20Zw8afxTnY2f4b7hmhn/z8aC2kfArVQlAhFBw==",
"license": "MIT",
"engines": {
"node": ">=20.18.1"
}
},
"node_modules/undici-types": {
"version": "5.26.5",
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz",

1
package.json
View File

@@ -58,7 +58,6 @@
"tar-stream": "^3.1.7",
"tldjs": "^2.3.1",
"ua-parser-js": "^2.0.2",
"undici": "^7.3.0",
"uuid": "^11.0.5",
"validator": "^13.12.0",
"ws": "^8.18.0",

120
src/superagent.js
View File

@@ -11,10 +11,12 @@ exports = module.exports = {
};
// IMPORTANT: do not require box code here . This is used by migration scripts
const { Agent } = require('undici'),
assert = require('assert'),
const assert = require('assert'),
consumers = require('node:stream/consumers'),
debug = require('debug')('box:superagent'),
fs = require('fs'),
http = require('http'),
https = require('https'),
path = require('path'),
safe = require('safetydance');
@@ -23,60 +25,89 @@ class Request {
assert.strictEqual(typeof url, 'string');
this.url = new URL(url);
this.fetchOptions = {
this.options = {
method,
headers: new Headers(),
redirect: 'follow'
headers: {},
signal: null // set for timeouts
};
this.okFunc = (response) => response.ok;
this.okFunc = ({ status }) => status >=200 && status <= 299;
this.timer = { timeout: 0, id: null, controller: null };
this.retryCount = 0;
this.body = null;
this.redirectCount = 5;
}
async _makeRequest() {
const response = await fetch(this.url.toString(), this.fetchOptions); // will only throw with network error or bad scheme
async _handleResponse(url, response) {
// const contentLength = response.headers['content-length'];
// if (!contentLength || contentLength > 5*1024*1024*1024) throw new Error(`Response size unknown or too large: ${contentLength}`);
const [consumeError, data] = await safe(consumers.buffer(response)); // have to drain response
if (consumeError) throw new Error(`Error consuming body stream: ${consumeError.message}`);
if (!response.complete) throw new Error('Incomplete response');
const contentType = response.headers['content-type'];
const result = {
url: response.url, // can be different from original when response.redirected
status: response.status,
headers: Object.fromEntries(response.headers.entries()),
url: new URL(response.headers['location'] || '', url),
status: response.statusCode,
headers: response.headers,
body: null,
text: null
};
const contentType = response.headers.get('Content-Type');
const data = await response.arrayBuffer();
if (contentType?.includes('application/json')) {
result.text = Buffer.from(data).toString('utf8');
if (data.byteLength !== 0) result.body = JSON.parse(result.text);
result.text = data.toString('utf8');
if (data.byteLength !== 0) result.body = safe.JSON.parse(result.text) || {};
} else if (contentType?.includes('application/x-www-form-urlencoded')) {
result.text = Buffer.from(data).toString('utf8');
result.text = data.toString('utf8');
const searchParams = new URLSearchParams(data);
result.body = Object.fromEntries(searchParams.entries());
} else if (!contentType) {
result.body = Buffer.from(data);
} else if (!contentType || contentType.startsWith('text/')) {
result.body = data;
result.text = result.body.toString('utf8');
} else {
result.body = Buffer.from(data);
result.body = data;
result.text = `<binary data (${data.byteLength} bytes)>`;
}
if (!this.okFunc(response)) {
const error = new Error(`${response.status} ${response.statusText}`);
Object.assign(error, result);
throw error;
}
return result;
}
async _makeRequest(url) {
return new Promise((resolve, reject) => {
const proto = url.protocol === 'https:' ? https : http;
const request = proto.request(url, this.options); // ClientRequest
request.on('error', reject); // network error, dns error
request.on('response', async (response) => {
const [error, result] = await safe(this._handleResponse(url, response));
if (error) reject(error); else resolve(result);
});
if (this.body) request.write(this.body, 'utf8');
request.end();
});
}
async _start() {
let error, response;
let error;
for (let i = 0; i < this.retryCount+1; i++) {
if (this.timer.timeout) this.timer.id = setTimeout(() => this.timer.controller.abort(), this.timer.timeout);
debug(`${this.fetchOptions.method} ${this.url.toString()}` + (i ? ` try ${i+1}` : ''));
[error, response] = await safe(this._makeRequest());
if (error) debug(`${this.fetchOptions.method} ${this.url.toString()} ${error.message}`);
debug(`${this.options.method} ${this.url.toString()}` + (i ? ` try ${i+1}` : ''));
let response, url = this.url;
for (let redirects = 0; redirects < this.redirectCount+1; redirects++) {
[error, response] = await safe(this._makeRequest(url));
if (error || (response.status < 300 || response.status > 399) || (this.options.method !== 'GET')) break;
url = response.url; // follow
}
if (!error && !this.okFunc({ status: response.status })) {
error = new Error(`${response.status} ${http.STATUS_CODES[response.status]}`);
Object.assign(error, response);
}
if (error) debug(`${this.options.method} ${this.url.toString()} ${error.message}`);
if (this.timer.timeout) clearTimeout(this.timer.id);
if (!error) return response;
}
@@ -85,7 +116,7 @@ class Request {
}
set(name, value) {
this.fetchOptions.headers.set(name, value);
this.options.headers[name.toLowerCase()] = value;
return this;
}
@@ -95,17 +126,19 @@ class Request {
}
redirects(count) {
this.fetchOptions.redirect = count ? 'follow' : 'manual'; // 'error' makes fetch throw
this.redirectCount = count;
return this;
}
send(data) {
const contentType = this.fetchOptions.headers.get('Content-Type');
const contentType = this.options.headers['content-type'];
if (!contentType || contentType === 'application/json') {
this.fetchOptions.headers.set('Content-Type', 'application/json');
this.fetchOptions.body = JSON.stringify(data);
this.options.headers['content-type'] = 'application/json';
this.body = JSON.stringify(data);
this.options.headers['content-length'] = Buffer.byteLength(this.body, 'utf8');
} else if (contentType === 'application/x-www-form-urlencoded') {
this.fetchOptions.body = new URLSearchParams(data);
this.body = (new URLSearchParams(data)).toString();
this.options.headers['content-length'] = Buffer.byteLength(this.body, 'utf8');
}
return this;
}
@@ -113,7 +146,7 @@ class Request {
timeout(msecs) {
this.timer.controller = new AbortController();
this.timer.timeout = msecs;
this.fetchOptions.signal = this.timer.controller.signal;
this.options.signal = this.timer.controller.signal;
return this;
}
@@ -128,12 +161,7 @@ class Request {
}
disableTLSCerts() {
this.fetchOptions.dispatcher = new Agent({
connect: {
rejectUnauthorized: false,
},
});
return this;
this.options.rejectUnauthorized = true;
}
auth(username, password) {
@@ -142,10 +170,10 @@ class Request {
}
attach(name, filepath) {
if (!(this.fetchOptions.body instanceof FormData)) this.fetchOptions.body = new FormData();
if (!(this.options.body instanceof FormData)) this.options.body = new FormData();
const data = fs.readFileSync(filepath);
this.fetchOptions.body.append(name, new Blob([data]), path.basename(filepath));
this.fetchOptions.headers.delete('Content-Type'); // explicitly remove it. without it boundary won't make it to the header!
this.options.body.append(name, new Blob([data]), path.basename(filepath));
delete this.options.headers['content-type']; // explicitly remove it. without it boundary won't make it to the header!
return this;
}

30
src/test/superagent-test.js
View File

@@ -12,9 +12,37 @@ describe('Superagent', function () {
await superagent.get('https://www.cloudron.io').set('User-Agent', 'Mozilla').timeout(10*1000);
});
it('cannot get invalid URL', async function () {
const [error] = await safe(superagent.get('htt://www.cloudron.io'));
expect(error).to.be.ok();
});
it('cannot get non-existent domain', async function () {
const [error] = await safe(superagent.get('https://www.cloudron.io.nxdomain'));
expect(error).to.be.ok();
});
it('throws for 404', async function () {
const [error] = await safe(superagent.get('https://www.cloudron.io/no-such-page'));
expect(error).to.be.ok();
expect(error.status).to.be(404);
expect(error.text).to.be.a('string');
});
it('can catch a 404', async function () {
const response = await superagent.get('https://www.cloudron.io/no-such-page').ok(({status}) => status === 404);
expect(response.status).to.be(404);
expect(response.text).to.be.a('string');
});
it('did parse json', async function () {
const response = await superagent.get('https://ipv4.api.cloudron.io/api/v1/helper/public_ip');
expect(response.body.ip).to.be.ok();
});
it('follows redirect', async function () {
const response = await superagent.get('https://cloudron.io').set('User-Agent', 'Mozilla').timeout(10*1000);
expect(response.url).to.be('https://www.cloudron.io/');
expect(response.url.toString()).to.be('https://www.cloudron.io/');
});
it('can disable redirects', async function () {