diff --git a/package-lock.json b/package-lock.json index d20c82794..b7a6e06f8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -39,7 +39,7 @@ "moment": "^2.30.1", "moment-timezone": "^0.6.0", "multiparty": "^4.2.3", - "mysql": "^2.18.1", + "mysql2": "^3.14.1", "nodemailer": "^7.0.3", "oidc-provider": "^9.1.3", "ovh": "^2.0.3", @@ -5544,10 +5544,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/isarray": { - "version": "1.0.0", - "license": "MIT" - }, "node_modules/isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", @@ -6400,30 +6396,10 @@ "version": "0.0.8", "license": "ISC" }, - "node_modules/mysql": { - "version": "2.18.1", - "license": "MIT", - "dependencies": { - "bignumber.js": "9.0.0", - "readable-stream": "2.3.7", - "safe-buffer": "5.1.2", - "sqlstring": "2.3.1" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/mysql/node_modules/bignumber.js": { - "version": "9.0.0", - "license": "MIT", - "engines": { - "node": "*" - } - }, "node_modules/mysql2": { - "version": "3.12.0", - "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.12.0.tgz", - "integrity": "sha512-C8fWhVysZoH63tJbX8d10IAoYCyXy4fdRFz2Ihrt9jtPILYynFEKUUzpp1U7qxzDc3tMbotvaBH+sl6bFnGZiw==", + "version": "3.14.1", + "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.14.1.tgz", + "integrity": "sha512-7ytuPQJjQB8TNAYX/H2yhL+iQOnIBjAMam361R7UAL0lOVXWjtdrmoL9HYKqKoLp/8UUTRcvo1QPvK9KL7wA8w==", "license": "MIT", "dependencies": { "aws-ssl-profiles": "^1.1.1", @@ -6904,10 +6880,6 @@ "node": ">= 0.8.0" } }, - "node_modules/process-nextick-args": { - "version": "2.0.1", - "license": "MIT" - }, "node_modules/prompt": { "version": "1.1.0", "license": "MIT", @@ -7206,19 +7178,6 @@ "node": ">=0.8" } }, - "node_modules/readable-stream": { - "version": "2.3.7", - "license": "MIT", - "dependencies": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.3", - "isarray": "~1.0.0", - "process-nextick-args": "~2.0.0", - "safe-buffer": "~5.1.1", - "string_decoder": "~1.1.1", - "util-deprecate": "~1.0.1" - } - }, "node_modules/readdirp": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-4.1.2.tgz", @@ -7634,13 +7593,6 @@ "integrity": "sha512-Q5thBSxp5t8WPTTJQS59LrGqOZqOsrhDGDVm8azCqIBjSBd7nd9o2PM+mDulQQkh8h//4U6hFZnc/mul8t5pWQ==", "license": "ISC" }, - "node_modules/sqlstring": { - "version": "2.3.1", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, "node_modules/ssh2": { "version": "1.16.0", "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-1.16.0.tgz", diff --git a/package.json b/package.json index 80fc72aeb..499081a27 100644 --- a/package.json +++ b/package.json @@ -43,7 +43,7 @@ "moment": "^2.30.1", "moment-timezone": "^0.6.0", "multiparty": "^4.2.3", - "mysql": "^2.18.1", + "mysql2": "^3.14.1", "nodemailer": "^7.0.3", "oidc-provider": "^9.1.3", "ovh": "^2.0.3", diff --git a/src/database.js b/src/database.js index bb1376b7d..585507e4a 100644 --- a/src/database.js +++ b/src/database.js @@ -13,12 +13,11 @@ exports = module.exports = { }; const assert = require('assert'), - async = require('async'), BoxError = require('./boxerror.js'), constants = require('./constants.js'), debug = require('debug')('box:database'), execSync = require('child_process').execSync, - mysql = require('mysql'), + mysql = require('mysql2/promise'), safe = require('safetydance'), shell = require('./shell.js')('database'); @@ -43,7 +42,6 @@ async function initialize() { // https://github.com/mysqljs/mysql#pool-options gConnectionPool = mysql.createPool({ connectionLimit: 5, - acquireTimeout: 60000, connectTimeout: 60000, host: gDatabase.hostname, user: gDatabase.username, @@ -54,30 +52,34 @@ async function initialize() { waitForConnections: true, // getConnection() will wait until a connection is avaiable ssl: false, timezone: 'Z', // mysql follows the SYSTEM timezone. on Cloudron, this is UTC - charset: 'utf8mb4' + charset: 'utf8mb4', + jsonStrings: true, // for JSON types, JSONARRAYAGG will return string instead of JSON }); - gConnectionPool.on('connection', function (connection) { - debug('connected'); + // run one time setup commands on new connections. connections are reused and so we cannot use 'acquire' event of the pool + gConnectionPool.on('connection', async function (connection) { // https://github.com/sidorares/node-mysql2/issues/565, 567 + debug('new connection'); - // connection objects are re-used. so we have to attach to the event here (once) to prevent crash - // note the pool also has an 'acquire' event but that is called whenever we do a getConnection() - connection.on('error', (error) => debug(`Connection ${connection.threadId} error: ${error.message} ${error.code}`)); + const conn = connection.promise(); // convert PoolConnection to PromisePoolConnection - connection.query(`USE ${gDatabase.name}`); - connection.query('SET SESSION sql_mode = \'strict_all_tables\''); - // GROUP_CONCAT has only 1024 default. we use it in the groups API which doesn't support pagination yet - // a uuid v4 is 36 in length. so the value below provides for roughly 10k users - connection.query('SET SESSION group_concat_max_len = 360000'); + try { + // await connection.query('SET NAMES utf8mb4 COLLATE utf8mb4_bin'); + await conn.query('SET SESSION sql_mode = \'strict_all_tables\''); // disable type coercion etc + // GROUP_CONCAT has only 1024 default. we use it in the groups API which doesn't support pagination yet + // a uuid v4 is 36 in length. so the value below provides for roughly 10k users + await conn.query('SET SESSION group_concat_max_len = 360000'); + } catch (error) { + debug(`failed to init new db connection ${connection.threadId}:`, error); // only log. we will let the app handle the exception when it calls query()/transaction() + } }); } async function uninitialize() { if (!gConnectionPool) return; - gConnectionPool.end(); + await safe(gConnectionPool.end(), { debug }); gConnectionPool = null; - debug('connection closed'); + debug('pool closed'); } async function clear() { @@ -89,53 +91,35 @@ async function clear() { await transaction(queries); } -async function query() { +async function query(...args) { assert.notStrictEqual(gConnectionPool, null, 'Database connection is already closed'); - return new Promise((resolve, reject) => { - const args = Array.prototype.slice.call(arguments); - - args.push(function queryCallback(error, result) { - if (error) return reject(new BoxError(BoxError.DATABASE_ERROR, error, { code: error.code, sqlMessage: error.sqlMessage || null })); - - resolve(result); - }); - - gConnectionPool.query.apply(gConnectionPool, args); // this is same as getConnection/query/release - }); + const [error, result] = await safe(gConnectionPool.query(...args)); // this is same as getConnection/query/release + if (error) throw new BoxError(BoxError.DATABASE_ERROR, error, { code: error.code, sqlMessage: error.sqlMessage || null }); + return result[0]; // the promise version returns a tuple of [rows, fields] } async function transaction(queries) { assert(Array.isArray(queries)); - return new Promise((resolve, reject) => { - gConnectionPool.getConnection(function (error, connection) { - if (error) return reject(new BoxError(BoxError.DATABASE_ERROR, error, { code: error.code, sqlMessage: error.sqlMessage })); + const [error, connection] = await safe(gConnectionPool.getConnection()); + if (error) throw new BoxError(BoxError.DATABASE_ERROR, error, { code: error.code, sqlMessage: error.sqlMessage }); - const releaseConnection = (error) => { - connection.release(); - reject(new BoxError(BoxError.DATABASE_ERROR, error, { code: error.code, sqlMessage: error.sqlMessage || null })); - }; - - connection.beginTransaction(function (error) { - if (error) return releaseConnection(error); - - async.mapSeries(queries, function iterator(query, done) { - connection.query(query.query, query.args, done); - }, function seriesDone(error, results) { - if (error) return connection.rollback(() => releaseConnection(error)); - - connection.commit(function (error) { - if (error) return connection.rollback(() => releaseConnection(error)); - - connection.release(); - - resolve(results); - }); - }); - }); - }); - }); + try { + await connection.beginTransaction(); + const results = []; + for (const query of queries) { + const [rows /*, fields */] = await connection.query(query.query, query.args); + results.push(rows); + } + await connection.commit(); + connection.release(); // no await! + return results; + } catch (error) { + await safe(connection.rollback(), { debug }); + connection.release(); // no await! + throw new BoxError(BoxError.DATABASE_ERROR, error, { code: error.code, sqlMessage: error.sqlMessage || null }); + } } async function importFromFile(file) { diff --git a/src/eventlog.js b/src/eventlog.js index a6d6d8f6d..668579e96 100644 --- a/src/eventlog.js +++ b/src/eventlog.js @@ -114,7 +114,7 @@ exports = module.exports = { const assert = require('assert'), database = require('./database.js'), debug = require('debug')('box:eventlog'), - mysql = require('mysql'), + mysql = require('mysql2'), notifications = require('./notifications.js'), safe = require('safetydance'), uuid = require('uuid'); diff --git a/src/mail.js b/src/mail.js index e92c297e3..42eb46883 100644 --- a/src/mail.js +++ b/src/mail.js @@ -69,7 +69,7 @@ const assert = require('assert'), eventlog = require('./eventlog.js'), mailer = require('./mailer.js'), mailServer = require('./mailserver.js'), - mysql = require('mysql'), + mysql = require('mysql2'), net = require('net'), network = require('./network.js'), nodemailer = require('nodemailer'), diff --git a/src/test/database-test.js b/src/test/database-test.js index decd8a577..7a16b6ee9 100644 --- a/src/test/database-test.js +++ b/src/test/database-test.js @@ -1,12 +1,10 @@ -/* global it:false */ -/* global describe:false */ -/* global before:false */ +/* global it, describe, before, after */ 'use strict'; -const database = require('../database'), +const BoxError = require('../boxerror.js'), + database = require('../database'), expect = require('expect.js'), - fs = require('fs'), safe = require('safetydance'); describe('database', function () { @@ -24,7 +22,54 @@ describe('database', function () { }); }); - describe('importFromFile', function () { + describe('query', function () { + before(async () => { await database.initialize(); }); + after(async () => { await database.uninitialize(); }); + + it('basic query', async function () { + const result = await database.query('SELECT COUNT(*) as count FROM apps'); + expect(result[0].count).to.be(0); + }); + + it('bad queries', async function () { + const queries = [ + 'SELECT COUNT(*) FROM oops', // bad table + 'SELECT', // incomplete + ]; + + for (const query of queries) { + const [error] = await safe(database.query(query)); + expect(error.reason).to.be(BoxError.DATABASE_ERROR); + expect(error.sqlMessage).to.be.a('string'); + } + }); + }); + + describe('transaction', function () { + before(async () => { await database.initialize(); }); + after(async () => { await database.uninitialize(); }); + + it('basic transaction', async function () { + const results = await database.transaction([ + { query: 'SELECT COUNT(*) as count FROM apps', args: [] }, + { query: 'SELECT COUNT(*) as count FROM settings', args: [] }, + ]); + expect(results[0][0].count).to.be(0); + expect(results[1][0].count).to.be(0); + }); + + it('bad transaction', async function () { + const [error] = await safe(database.transaction([ + { query: 'SELECT COUNT(*) as count FROM apps', args: [] }, + { query: 'stuff', args: [] }, + ])); + + expect(error.reason).to.be(BoxError.DATABASE_ERROR); + expect(error.sqlMessage).to.be.a('string'); + }); + }); + + describe('import/export', function () { before(async function () { await database.initialize(); await database._clear(); @@ -42,5 +87,9 @@ describe('database', function () { it('can import from file', async function () { await database.importFromFile('/tmp/box.mysqldump'); }); + + it('can uninitialize database', async function () { + await database.uninitialize(); + }); }); }); diff --git a/src/users.js b/src/users.js index 963176fda..aca657fbb 100644 --- a/src/users.js +++ b/src/users.js @@ -89,7 +89,7 @@ const appPasswords = require('./apppasswords.js'), hat = require('./hat.js'), mail = require('./mail.js'), mailer = require('./mailer.js'), - mysql = require('mysql'), + mysql = require('mysql2'), notifications = require('./notifications'), oidcClients = require('./oidcclients.js'), qrcode = require('qrcode'),