diff --git a/package-lock.json b/package-lock.json index d8b24af5b..114daae25 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,6 +13,7 @@ "@aws-sdk/lib-storage": "^3.928.0", "@cloudron/connect-lastmile": "^2.3.0", "@cloudron/manifest-format": "^5.29.0", + "@cloudron/pipework": "^1.0.1", "@cloudron/superagent": "^1.0.1", "@google-cloud/dns": "^5.3.1", "@google-cloud/storage": "^7.17.3", @@ -1157,6 +1158,12 @@ "validator": "^13.15.15" } }, + "node_modules/@cloudron/pipework": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@cloudron/pipework/-/pipework-1.0.1.tgz", + "integrity": "sha512-T1LART+O7CoXMYDvPXVEgqtmb5d63H0wKB9jWAfjWIzqS0IOFMeawUBQaPKTO7aH8vNFEwNfU8XYK4vnJpCZ4w==", + "license": "ISC" + }, "node_modules/@cloudron/superagent": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/@cloudron/superagent/-/superagent-1.0.1.tgz", diff --git a/package.json b/package.json index f58b19e48..d813f368e 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,7 @@ "@aws-sdk/lib-storage": "^3.928.0", "@cloudron/connect-lastmile": "^2.3.0", "@cloudron/manifest-format": "^5.29.0", + "@cloudron/pipework": "^1.0.1", "@cloudron/superagent": "^1.0.1", "@google-cloud/dns": "^5.3.1", "@google-cloud/storage": "^7.17.3", diff --git a/src/services.js b/src/services.js index b0ddc71d2..7c1194b4c 100644 --- a/src/services.js +++ b/src/services.js @@ -54,7 +54,6 @@ const addonConfigs = require('./addonconfigs.js'), eventlog = require('./eventlog.js'), fs = require('node:fs'), hat = require('./hat.js'), - http = require('node:http'), infra = require('./infra_version.js'), logs = require('./logs.js'), mail = require('./mail.js'), @@ -63,7 +62,7 @@ const addonConfigs = require('./addonconfigs.js'), os = require('node:os'), path = require('node:path'), paths = require('./paths.js'), - { pipeline } = require('node:stream'), + { pipeFileToRequest, pipeRequestToFile } = require('@cloudron/pipework'), promiseRetry = require('./promise-retry.js'), safe = require('safetydance'), semver = require('semver'), @@ -652,8 +651,6 @@ async function backupAddons(app, addons) { assert.strictEqual(typeof app, 'object'); assert(!addons || typeof addons === 'object'); - debug('backupAddons'); - if (!addons) return; debug('backupAddons: backing up %j', Object.keys(addons)); @@ -669,8 +666,6 @@ async function clearAddons(app, addons) { assert.strictEqual(typeof app, 'object'); assert(!addons || typeof addons === 'object'); - debug('clearAddons'); - if (!addons) return; debug('clearAddons: clearing %j', Object.keys(addons)); @@ -686,8 +681,6 @@ async function restoreAddons(app, addons) { assert.strictEqual(typeof app, 'object'); assert(!addons || typeof addons === 'object'); - debug('restoreAddons'); - if (!addons) return; debug('restoreAddons: restoring %j', Object.keys(addons)); @@ -713,19 +706,19 @@ async function importAppDatabase(app, addon) { async function importDatabase(addon) { assert.strictEqual(typeof addon, 'string'); - debug(`importDatabase: Importing ${addon}`); + debug(`importDatabase: importing ${addon}`); const allApps = await apps.list(); for (const app of allApps) { if (!app.manifest.addons || !(addon in app.manifest.addons)) continue; // app doesn't use the addon - debug(`importDatabase: Importing addon ${addon} of app ${app.id}`); + debug(`importDatabase: importing addon ${addon} of app ${app.id}`); const [error] = await safe(importAppDatabase(app, addon)); if (!error) continue; - debug(`importDatabase: Error importing ${addon} of app ${app.id}. Marking as errored. %o`, error); + debug(`importDatabase: error importing ${addon} of app ${app.id}. Marking as errored. %o`, error); // FIXME: there is no way to 'repair' if we are here. we need to make a separate apptask that re-imports db // not clear, if repair workflow should be part of addon or per-app await safe(apps.update(app.id, { installationState: apps.ISTATE_ERROR, error: { message: error.message } })); @@ -737,10 +730,10 @@ async function importDatabase(addon) { async function exportDatabase(addon) { assert.strictEqual(typeof addon, 'string'); - debug(`exportDatabase: Exporting ${addon}`); + debug(`exportDatabase: exporting ${addon}`); if (fs.existsSync(path.join(paths.ADDON_CONFIG_DIR, `exported-${addon}`))) { - debug(`exportDatabase: Already exported addon ${addon} in previous run`); + debug(`exportDatabase: already exported addon ${addon} in previous run`); return; } @@ -750,11 +743,11 @@ async function exportDatabase(addon) { if (!app.manifest.addons || !(addon in app.manifest.addons)) continue; // app doesn't use the addon if (app.installationState === apps.ISTATE_ERROR) continue; // missing db causes crash in old app addon containers - debug(`exportDatabase: Exporting addon ${addon} of app ${app.id}`); + debug(`exportDatabase: exporting addon ${addon} of app ${app.id}`); const [error] = await safe(ADDONS[addon].backup(app, app.manifest.addons[addon])); if (error) { - debug(`exportDatabase: Error exporting ${addon} of app ${app.id}. %o`, error); + debug(`exportDatabase: error exporting ${addon} of app ${app.id}. %o`, error); // for errored apps, we can ignore if export had an error if (app.installationState === apps.ISTATE_ERROR) continue; throw error; @@ -1282,8 +1275,8 @@ async function setupMySql(app, options) { .send(data) .ok(() => true)); - if (networkError) throw new BoxError(BoxError.ADDONS_ERROR, `Network error setting up mysql: ${networkError.message}`); - if (response.status !== 201) throw new BoxError(BoxError.ADDONS_ERROR, `Error setting up mysql. Status code: ${response.status} message: ${response.body.message}`); + if (networkError) throw new BoxError(BoxError.ADDONS_ERROR, `Network error setting up MySQL: ${networkError.message}`); + if (response.status !== 201) throw new BoxError(BoxError.ADDONS_ERROR, `Error setting up MySQL. Status code: ${response.status} message: ${response.body.message}`); let env = [ { name: 'CLOUDRON_MYSQL_USERNAME', value: data.username }, @@ -1316,8 +1309,8 @@ async function clearMySql(app, options) { const [networkError, response] = await safe(superagent.post(`http://${result.ip}:3000/` + (options.multipleDatabases ? 'prefixes' : 'databases') + `/${database}/clear?access_token=${result.token}`) .ok(() => true)); - if (networkError) throw new BoxError(BoxError.ADDONS_ERROR, `Network error clearing mysql: ${networkError.message}`); - if (response.status !== 200) throw new BoxError(BoxError.ADDONS_ERROR, `Error clearing mysql. Status code: ${response.status} message: ${response.body.message}`); + if (networkError) throw new BoxError(BoxError.ADDONS_ERROR, `Network error clearing MySQL: ${networkError.message}`); + if (response.status !== 200) throw new BoxError(BoxError.ADDONS_ERROR, `Error clearing MySQL. Status code: ${response.status} message: ${response.body.message}`); } async function teardownMySql(app, options) { @@ -1332,74 +1325,12 @@ async function teardownMySql(app, options) { const [networkError, response] = await safe(superagent.del(`http://${result.ip}:3000/` + (options.multipleDatabases ? 'prefixes' : 'databases') + `/${database}?access_token=${result.token}&username=${username}`) .ok(() => true)); - if (networkError) throw new BoxError(BoxError.ADDONS_ERROR, `Error tearing down mysql: ${networkError.message}`); - if (response.status !== 200) throw new BoxError(BoxError.ADDONS_ERROR, `Error tearing down mysql. Status code: ${response.status} message: ${response.body.message}`); + if (networkError) throw new BoxError(BoxError.ADDONS_ERROR, `Error tearing down MySQL: ${networkError.message}`); + if (response.status !== 200) throw new BoxError(BoxError.ADDONS_ERROR, `Error tearing down MySQL. Status code: ${response.status} message: ${response.body.message}`); await addonConfigs.unset(app.id, 'mysql'); } -function pipeRequestToFile(url, filename) { - assert.strictEqual(typeof url, 'string'); - assert.strictEqual(typeof filename, 'string'); - - return new Promise((resolve, reject) => { - const writeStream = fs.createWriteStream(filename); - const doReject = (error) => { writeStream.destroy(); reject(error); }; - - const request = http.request(url, { method: 'POST' }); // ClientRequest - request.setTimeout(4 * 60 * 60 * 1000, () => { - debug('pipeRequestToFile: timeout - connect or post-connect idle timeout'); - request.destroy(); // connect OR post-connect idle timeout - doReject(new BoxError(BoxError.NETWORK_ERROR, 'Request timedout')); - }); - - request.on('error', (error) => doReject(new BoxError(BoxError.NETWORK_ERROR, `Could not pipe ${url} to ${filename}: ${error.message}`))); // network error, dns error - request.on('response', (response) => { - debug(`pipeRequestToFile: connected with status code ${response.statusCode}`); - if (response.statusCode !== 200) { - response.resume(); // drain the response - return doReject(new BoxError(BoxError.ADDONS_ERROR, `Unexpected response code or HTTP error when piping ${url} to ${filename}: status ${response.statusCode}`)); - } - - pipeline(response, writeStream, (error) => { - if (error) return reject(new BoxError(BoxError.ADDONS_ERROR, `Error piping ${url} to ${filename}: ${error.message}`)); - if (!response.complete) return reject(new BoxError(BoxError.ADDONS_ERROR, `Response not complete when piping ${url} to ${filename}`)); - resolve(); - }); - }); - request.end(); // make the request - }); -} - -function pipeFileToRequest(filename, url) { - assert.strictEqual(typeof filename, 'string'); - assert.strictEqual(typeof url, 'string'); - - return new Promise((resolve, reject) => { - const readStream = fs.createReadStream(filename); - const doReject = (error) => { readStream.destroy(); reject(error); }; - - const request = http.request(url, { method: 'POST' }); // ClientRequest - request.setTimeout(4 * 60 * 60 * 1000, () => { - debug('pipeFileToRequest: timeout - connect or post-connect idle timeout'); - request.destroy(); - doReject(new BoxError(BoxError.NETWORK_ERROR, 'Request timedout')); - }); - request.on('response', (response) => { - debug(`pipeFileToRequest: request completed with status code ${response.statusCode}`); - response.resume(); // drain the response - if (response.statusCode !== 200) return doReject(new BoxError(BoxError.ADDONS_ERROR, `Unexpected response code or HTTP error when piping ${filename} to ${url}: status ${response.statusCode} complete ${response.complete}`)); - resolve(); - }); - - debug(`pipeFileToRequest: piping ${filename} to ${url}`); - pipeline(readStream, request, function (error) { - if (error) return reject(new BoxError(BoxError.ADDONS_ERROR, `Error piping file ${filename} to request ${url}`)); - debug(`pipeFileToRequest: piped ${filename} to ${url}`); // now we have to wait for 'response' above - }); - }); -} - async function backupMySql(app, options) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); @@ -1411,7 +1342,8 @@ async function backupMySql(app, options) { const result = await getContainerDetails('mysql', 'CLOUDRON_MYSQL_TOKEN'); const url = `http://${result.ip}:3000/` + (options.multipleDatabases ? 'prefixes' : 'databases') + `/${database}/backup?access_token=${result.token}`; - await pipeRequestToFile(url, dumpPath('mysql', app.id)); + const [error] = await safe(pipeRequestToFile(url, dumpPath('mysql', app.id))); + if (error) throw new BoxError(BoxError.ADDONS_ERROR, `Error backing up MySQL: ${error.message}`); } async function restoreMySql(app, options) { @@ -1425,7 +1357,8 @@ async function restoreMySql(app, options) { const result = await getContainerDetails('mysql', 'CLOUDRON_MYSQL_TOKEN'); const url = `http://${result.ip}:3000/` + (options.multipleDatabases ? 'prefixes' : 'databases') + `/${database}/restore?access_token=${result.token}`; - await pipeFileToRequest(dumpPath('mysql', app.id), url); + const [error] = await safe(pipeFileToRequest(dumpPath('mysql', app.id), url)); + if (error) throw new BoxError(BoxError.ADDONS_ERROR, `MySQL restore failed. This may require more memory. Check logs in the Services view. Details: ${error.message}`); } function postgreSqlNames(appId) { @@ -1506,8 +1439,8 @@ async function setupPostgreSql(app, options) { const [networkError, response] = await safe(superagent.post(`http://${result.ip}:3000/databases?access_token=${result.token}`) .send(data) .ok(() => true)); - if (networkError) throw new BoxError(BoxError.ADDONS_ERROR, `Network error setting up postgresql: ${networkError.message}`); - if (response.status !== 201) throw new BoxError(BoxError.ADDONS_ERROR, `Error setting up postgresql. Status code: ${response.status} message: ${response.body.message}`); + if (networkError) throw new BoxError(BoxError.ADDONS_ERROR, `Network error setting up PostgreSQL: ${networkError.message}`); + if (response.status !== 201) throw new BoxError(BoxError.ADDONS_ERROR, `Error setting up PostgreSQL. Status code: ${response.status} message: ${response.body.message}`); const env = [ { name: 'CLOUDRON_POSTGRESQL_URL', value: `postgres://${data.username}:${data.password}@postgresql/${data.database}` }, @@ -1535,8 +1468,8 @@ async function clearPostgreSql(app, options) { const [networkError, response] = await safe(superagent.post(`http://${result.ip}:3000/databases/${database}/clear?access_token=${result.token}&username=${username}&locale=${locale}`) .ok(() => true)); - if (networkError) throw new BoxError(BoxError.ADDONS_ERROR, `Network error clearing postgresql: ${networkError.message}`); - if (response.status !== 200) throw new BoxError(BoxError.ADDONS_ERROR, `Error clearing postgresql. Status code: ${response.status} message: ${response.body.message}`); + if (networkError) throw new BoxError(BoxError.ADDONS_ERROR, `Network error clearing PostgreSQL: ${networkError.message}`); + if (response.status !== 200) throw new BoxError(BoxError.ADDONS_ERROR, `Error clearing PostgreSQL. Status code: ${response.status} message: ${response.body.message}`); } async function teardownPostgreSql(app, options) { @@ -1549,8 +1482,8 @@ async function teardownPostgreSql(app, options) { const [networkError, response] = await safe(superagent.del(`http://${result.ip}:3000/databases/${database}?access_token=${result.token}&username=${username}`) .ok(() => true)); - if (networkError) throw new BoxError(BoxError.ADDONS_ERROR, `Network error tearing down postgresql: ${networkError.message}`); - if (response.status !== 200) throw new BoxError(BoxError.ADDONS_ERROR, `Error tearing down postgresql. Status code: ${response.status} message: ${response.body.message}`); + if (networkError) throw new BoxError(BoxError.ADDONS_ERROR, `Network error tearing down PostgreSQL: ${networkError.message}`); + if (response.status !== 200) throw new BoxError(BoxError.ADDONS_ERROR, `Error tearing down PostgreSQL. Status code: ${response.status} message: ${response.body.message}`); await addonConfigs.unset(app.id, 'postgresql'); } @@ -1564,7 +1497,8 @@ async function backupPostgreSql(app, options) { const { database } = postgreSqlNames(app.id); const result = await getContainerDetails('postgresql', 'CLOUDRON_POSTGRESQL_TOKEN'); - await pipeRequestToFile(`http://${result.ip}:3000/databases/${database}/backup?access_token=${result.token}`, dumpPath('postgresql', app.id)); + const [error] = await safe(pipeRequestToFile(`http://${result.ip}:3000/databases/${database}/backup?access_token=${result.token}`, dumpPath('postgresql', app.id))); + if (error) throw new BoxError(BoxError.ADDONS_ERROR, `Error backing up PostgreSQL: ${error.message}`); } async function restorePostgreSql(app, options) { @@ -1577,7 +1511,8 @@ async function restorePostgreSql(app, options) { const result = await getContainerDetails('postgresql', 'CLOUDRON_POSTGRESQL_TOKEN'); - await pipeFileToRequest(dumpPath('postgresql', app.id), `http://${result.ip}:3000/databases/${database}/restore?access_token=${result.token}&username=${username}`); + const [error] = await safe(pipeFileToRequest(dumpPath('postgresql', app.id), `http://${result.ip}:3000/databases/${database}/restore?access_token=${result.token}&username=${username}`)); + if (error) throw new BoxError(BoxError.ADDONS_ERROR, `PostgreSQL restore failed. This may require more memory. Check logs in the Services view. Details: ${error.message}`); } async function startMongodb(existingInfra) { @@ -1641,7 +1576,7 @@ async function setupMongoDb(app, options) { debug('Setting up mongodb'); - if (!await hasAVX()) throw new BoxError(BoxError.ADDONS_ERROR, 'Error setting up mongodb. CPU has no AVX support'); + if (!await hasAVX()) throw new BoxError(BoxError.ADDONS_ERROR, 'Error setting up MongoDB. CPU has no AVX support'); const existingPassword = await addonConfigs.getByName(app.id, 'mongodb', '%MONGODB_PASSWORD'); let database = await addonConfigs.getByName(app.id, 'mongodb', '%MONGODB_DATABASE'); @@ -1660,8 +1595,8 @@ async function setupMongoDb(app, options) { .send(data) .ok(() => true)); - if (networkError) throw new BoxError(BoxError.ADDONS_ERROR, `Network error setting up mongodb: ${networkError.message}`); - if (response.status !== 201) throw new BoxError(BoxError.ADDONS_ERROR, `Error setting up mongodb. Status code: ${response.status} message: ${response.body.message}`); + if (networkError) throw new BoxError(BoxError.ADDONS_ERROR, `Network error setting up MongoDB: ${networkError.message}`); + if (response.status !== 201) throw new BoxError(BoxError.ADDONS_ERROR, `Error setting up MongoDB. Status code: ${response.status} message: ${response.body.message}`); const env = [ { name: 'CLOUDRON_MONGODB_URL', value : `mongodb://${data.username}:${data.password}@mongodb:27017/${data.database}` }, @@ -1684,25 +1619,25 @@ async function clearMongodb(app, options) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); - if (!await hasAVX()) throw new BoxError(BoxError.ADDONS_ERROR, 'Error clearing mongodb. CPU has no AVX support'); + if (!await hasAVX()) throw new BoxError(BoxError.ADDONS_ERROR, 'Error clearing MongoDB. CPU has no AVX support'); const result = await getContainerDetails('mongodb', 'CLOUDRON_MONGODB_TOKEN'); const database = await addonConfigs.getByName(app.id, 'mongodb', '%MONGODB_DATABASE'); - if (!database) throw new BoxError(BoxError.NOT_FOUND, 'Error clearing mongodb. No database'); + if (!database) throw new BoxError(BoxError.NOT_FOUND, 'Error clearing MongoDB. No database'); const [networkError, response] = await safe(superagent.post(`http://${result.ip}:3000/databases/${database}/clear?access_token=${result.token}`) .ok(() => true)); - if (networkError) throw new BoxError(BoxError.ADDONS_ERROR, `Network error clearing mongodb: ${networkError.message}`); - if (response.status !== 200) throw new BoxError(BoxError.ADDONS_ERROR, `Error clearing mongodb. Status code: ${response.status} message: ${response.body.message}`); + if (networkError) throw new BoxError(BoxError.ADDONS_ERROR, `Network error clearing MongoDB: ${networkError.message}`); + if (response.status !== 200) throw new BoxError(BoxError.ADDONS_ERROR, `Error clearing MongoDB. Status code: ${response.status} message: ${response.body.message}`); } async function teardownMongoDb(app, options) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); - if (!await hasAVX()) throw new BoxError(BoxError.ADDONS_ERROR, 'Error tearing down mongodb. CPU has no AVX support'); + if (!await hasAVX()) throw new BoxError(BoxError.ADDONS_ERROR, 'Error tearing down MongoDB. CPU has no AVX support'); const result = await getContainerDetails('mongodb', 'CLOUDRON_MONGODB_TOKEN'); @@ -1712,8 +1647,8 @@ async function teardownMongoDb(app, options) { const [networkError, response] = await safe(superagent.del(`http://${result.ip}:3000/databases/${database}?access_token=${result.token}`) .ok(() => true)); - if (networkError) throw new BoxError(BoxError.ADDONS_ERROR, `Error tearing down mongodb: ${networkError.message}`); - if (response.status !== 200) throw new BoxError(BoxError.ADDONS_ERROR, `Error tearing down mongodb. Status code: ${response.status} message: ${response.body.message}`); + if (networkError) throw new BoxError(BoxError.ADDONS_ERROR, `Error tearing down MongoDB: ${networkError.message}`); + if (response.status !== 200) throw new BoxError(BoxError.ADDONS_ERROR, `Error tearing down MongoDB. Status code: ${response.status} message: ${response.body.message}`); addonConfigs.unset(app.id, 'mongodb'); } @@ -1724,14 +1659,15 @@ async function backupMongoDb(app, options) { debug('Backing up mongodb'); - if (!await hasAVX()) throw new BoxError(BoxError.ADDONS_ERROR, 'Error backing up mongodb. CPU has no AVX support'); + if (!await hasAVX()) throw new BoxError(BoxError.ADDONS_ERROR, 'Error backing up MongoDB. CPU has no AVX support'); const result = await getContainerDetails('mongodb', 'CLOUDRON_MONGODB_TOKEN'); const database = await addonConfigs.getByName(app.id, 'mongodb', '%MONGODB_DATABASE'); - if (!database) throw new BoxError(BoxError.NOT_FOUND, 'Error backing up mongodb. No database'); + if (!database) throw new BoxError(BoxError.NOT_FOUND, 'Error backing up MongoDB. No database'); - await pipeRequestToFile(`http://${result.ip}:3000/databases/${database}/backup?access_token=${result.token}`, dumpPath('mongodb', app.id)); + const [error] = await safe(pipeRequestToFile(`http://${result.ip}:3000/databases/${database}/backup?access_token=${result.token}`, dumpPath('mongodb', app.id))); + if (error) throw new BoxError(BoxError.ADDONS_ERROR, `Error backing up MongoDB: ${error.message}`); } async function restoreMongoDb(app, options) { @@ -1745,9 +1681,10 @@ async function restoreMongoDb(app, options) { const result = await getContainerDetails('mongodb', 'CLOUDRON_MONGODB_TOKEN'); const database = await addonConfigs.getByName(app.id, 'mongodb', '%MONGODB_DATABASE'); - if (!database) throw new BoxError(BoxError.NOT_FOUND, 'Error restoring mongodb. No database'); + if (!database) throw new BoxError(BoxError.NOT_FOUND, 'Error restoring MongoDB. No database'); - await pipeFileToRequest(dumpPath('mongodb', app.id), `http://${result.ip}:3000/databases/${database}/restore?access_token=${result.token}`); + const [error] = await safe(pipeFileToRequest(dumpPath('mongodb', app.id), `http://${result.ip}:3000/databases/${database}/restore?access_token=${result.token}`)); + if (error) throw new BoxError(BoxError.ADDONS_ERROR, `MongoDB restore failed. This may require more memory. Check logs in the Services view. Details: ${error.message}`); } async function statusMongodb() { @@ -1997,7 +1934,8 @@ async function backupRedis(app, options) { debug('Backing up redis'); const result = await getContainerDetails('redis-' + app.id, 'CLOUDRON_REDIS_TOKEN'); - await pipeRequestToFile(`http://${result.ip}:3000/backup?access_token=${result.token}`, dumpPath('redis', app.id)); + const [error] = await safe(pipeRequestToFile(`http://${result.ip}:3000/backup?access_token=${result.token}`, dumpPath('redis', app.id))); + if (error) throw new BoxError(BoxError.ADDONS_ERROR, `Error backing up Redis: ${error.message}`); } async function restoreRedis(app, options) { @@ -2010,7 +1948,8 @@ async function restoreRedis(app, options) { debug('Restoring redis'); const result = await getContainerDetails('redis-' + app.id, 'CLOUDRON_REDIS_TOKEN'); - await pipeFileToRequest(dumpPath('redis', app.id), `http://${result.ip}:3000/restore?access_token=${result.token}`); + const [error] = await safe(pipeFileToRequest(dumpPath('redis', app.id), `http://${result.ip}:3000/restore?access_token=${result.token}`)); + if (error) throw new BoxError(BoxError.ADDONS_ERROR, `Redis restore failed. This may require more memory. Check logs in the Services view. Details: ${error.message}`); } async function setupTls(app, options) {