Files
cloudron-box/src/test/common.js

266 lines
7.1 KiB
JavaScript
Raw Normal View History

2017-09-29 09:56:01 -07:00
'use strict';
2021-08-20 09:19:44 -07:00
const apps = require('../apps.js'),
2023-08-04 15:34:38 +05:30
appstore = require('../appstore.js'),
constants = require('../constants.js'),
cron = require('../cron.js'),
dashboard = require('../dashboard.js'),
2021-06-03 12:20:44 -07:00
database = require('../database.js'),
domains = require('../domains.js'),
2021-08-13 14:43:08 -07:00
expect = require('expect.js'),
2021-06-03 12:20:44 -07:00
fs = require('fs'),
remove global lock Currently, the update/apptask/fullbackup/platformstart take a global lock and cannot run in parallel. This causes situations where when a user tries to trigger an apptask, it says "waiting for backup to finish..." etc The solution is to let them run in parallel. We need a lock at the app level as app operations running in parallel would be bad (tm). In addition, the update task needs a lock just for the update part. We also need multi-process locks. Running tasks as processes is core to our "kill" strategy. Various inter process locks were explored: * node's IPC mechanism with process.send(). But this only works for direct node.js children. taskworker is run via sudo and the IPC does not work. * File lock using O_EXCL. Basic ideas to create lock files. While file creation can be done atomically, it becomes complicated to clean up lock files when the tasks crash. We need a way to know what locks were held by the crashing task. flock and friends are not built-into node.js * sqlite/redis were options but introduce additional deps * Settled on MySQL based locking. Initial plan was to have row locks or table locks. Each row is a kind of lock. While implementing, it was found that we need many types of locks (and not just update lock and app locks). For example, we need locks for each task type, so that only one task type is active at a time. * Instead of rows, we can just lock table and have a json blob in it. This hit a road block that LOCK TABLE is per session and our db layer cannot handle this easily! i.e when issing two db.query() it might use two different connections from the pool. We have to expose the connection, release connection etc. * Next idea was atomic blob update of the blob checking if old blob was same. This approach, was finally refined into a version field. Phew!
2024-12-07 14:35:45 +01:00
locks = require('../locks.js'),
2021-06-03 12:20:44 -07:00
mailer = require('../mailer.js'),
mailServer = require('../mailserver.js'),
2021-06-03 12:20:44 -07:00
nock = require('nock'),
2017-09-29 09:56:01 -07:00
path = require('path'),
2022-11-22 21:38:22 +01:00
paths = require('../paths.js'),
2021-06-03 12:20:44 -07:00
settings = require('../settings.js'),
2021-07-14 19:03:12 -07:00
tasks = require('../tasks.js'),
2023-05-14 10:53:50 +02:00
timers = require('timers/promises'),
2021-06-03 12:20:44 -07:00
users = require('../users.js');
2021-08-13 15:49:59 -07:00
const manifest = {
2021-06-03 12:20:44 -07:00
'id': 'io.cloudron.test',
'author': 'The Presidents Of the United States Of America',
'title': 'test title',
'description': 'test description',
'tagline': 'test rocks',
'website': 'http://test.cloudron.io',
'contactEmail': 'test@cloudron.io',
'version': '0.1.0',
'manifestVersion': 2,
2021-06-03 12:20:44 -07:00
'dockerImage': 'cloudron/test:25.2.0',
'healthCheckPath': '/',
'httpPort': 7777,
'tcpPorts': {
'ECHO_SERVER_PORT': {
'title': 'Echo Server Port',
'description': 'Echo server',
'containerPort': 7778
}
},
'addons': {
'oauth': { },
'redis': { },
'mysql': { },
'postgresql': { }
}
};
2022-11-23 14:36:57 +01:00
// copied from the proxy app CloudronManifest.json
const proxyAppManifest = {
2023-05-14 10:53:50 +02:00
'id': 'io.cloudron.builtin.appproxy',
'title': 'App Proxy',
'author': 'Cloudron Team',
'version': '1.0.0',
'upstreamVersion': '1.0.0',
'description': 'file://DESCRIPTION.md',
'tagline': 'Proxy an app through Cloudron',
'tags': [ 'proxy', 'external' ],
'healthCheckPath': '/',
'httpPort': 3000,
'minBoxVersion': '7.3.0',
'dockerImage': 'istobeignored',
'manifestVersion': 2,
'multiDomain': true,
'website': 'https://cloudron.io',
'documentationUrl': 'https://docs.cloudron.io/dashboard/#app-proxy',
'forumUrl': 'https://forum.cloudron.io',
'contactEmail': 'support@cloudron.io',
'icon': 'file://logo.png',
'addons': {},
'mediaLinks': [
'https://screenshots.cloudron.io/io.cloudron.builtin.appproxy/diagram.png'
],
'changelog': 'file://CHANGELOG.md'
2022-11-23 14:36:57 +01:00
};
2021-08-13 10:41:10 -07:00
const domain = {
2021-06-03 12:20:44 -07:00
domain: 'example.com',
zoneName: 'example.com',
2021-06-03 16:16:04 -07:00
provider: 'noop',
config: {},
2021-06-03 12:20:44 -07:00
fallbackCertificate: null,
2021-06-03 16:16:04 -07:00
tlsConfig: { provider: 'fallback' },
2021-06-03 12:20:44 -07:00
wellKnown: null
};
2021-08-20 09:19:44 -07:00
Object.freeze(domain);
2021-06-03 12:20:44 -07:00
2021-08-13 10:41:10 -07:00
const auditSource = { ip: '1.2.3.4' };
2021-06-03 12:20:44 -07:00
2021-08-13 10:41:10 -07:00
const admin = {
2021-06-28 15:15:28 -07:00
id: null,
2021-07-20 09:05:21 -07:00
username: 'testadmin',
2021-06-03 12:20:44 -07:00
password: 'secret123',
email: 'admin@me.com',
fallbackEmail: 'admin@external.com',
2021-06-03 12:20:44 -07:00
salt: 'morton',
createdAt: 'sometime back',
resetToken: '',
2021-07-20 09:05:21 -07:00
displayName: 'Administrator',
groupIds: [],
2021-06-03 12:20:44 -07:00
role: 'owner',
source: '',
2021-07-19 12:43:30 -07:00
avatar: constants.AVATAR_GRAVATAR,
active: true,
2021-06-03 12:20:44 -07:00
};
2021-08-13 10:41:10 -07:00
const user = {
2021-07-20 09:05:21 -07:00
id: null,
username: 'user',
2021-07-07 12:59:17 -07:00
password: '123secret',
2021-07-20 09:05:21 -07:00
email: 'user@me.com',
fallbackEmail: 'user@external.com',
2021-06-28 15:15:28 -07:00
role: 'user',
salt: 'morton',
createdAt: 'sometime back',
2021-07-20 09:05:21 -07:00
resetToken: '',
groupIds: [],
2021-07-20 09:05:21 -07:00
displayName: 'Normal User',
2021-06-28 15:15:28 -07:00
source: '',
2021-07-19 12:43:30 -07:00
avatar: constants.AVATAR_NONE,
active: true,
2021-06-28 15:15:28 -07:00
};
2021-08-13 10:41:10 -07:00
const app = {
2021-06-03 12:20:44 -07:00
id: 'appid',
appStoreId: 'appStoreId',
installationState: apps.ISTATE_PENDING_INSTALL,
runState: 'running',
subdomain: 'applocation',
2021-08-13 10:41:10 -07:00
domain: domain.domain,
fqdn: domain.domain + '.' + 'applocation',
2021-08-13 15:49:59 -07:00
manifest,
2021-06-03 12:20:44 -07:00
containerId: 'someid',
portBindings: {},
2021-06-03 12:20:44 -07:00
accessRestriction: null,
memoryLimit: 0,
2021-08-13 10:41:10 -07:00
mailboxDomain: domain.domain,
secondaryDomains: [],
redirectDomains: [],
2021-06-03 12:20:44 -07:00
aliasDomains: []
};
2021-08-20 09:19:44 -07:00
Object.freeze(app);
2017-09-29 09:56:01 -07:00
2022-11-23 14:36:57 +01:00
const proxyApp = {
id: 'proxyapptestid',
appStoreId: proxyAppManifest.id,
installationState: apps.ISTATE_PENDING_INSTALL,
runState: 'running',
subdomain: 'proxylocation',
upstreamUri: 'http://1.2.3.4:80',
domain: domain.domain,
fqdn: domain.domain + '.' + 'proxylocation',
manifest: proxyAppManifest,
2022-11-23 14:36:57 +01:00
containerId: '',
portBindings: {},
2022-11-23 14:36:57 +01:00
accessRestriction: null,
memoryLimit: 0,
mailboxDomain: domain.domain,
secondaryDomains: [],
redirectDomains: [],
aliasDomains: []
};
Object.freeze(proxyApp);
2017-09-29 09:56:01 -07:00
exports = module.exports = {
2021-06-03 12:20:44 -07:00
createTree,
2021-07-19 12:43:30 -07:00
domainSetup,
2021-08-13 17:22:28 -07:00
databaseSetup,
2021-06-03 12:20:44 -07:00
setup,
cleanup,
2021-08-13 14:43:08 -07:00
checkMails,
clearMailQueue,
2021-06-03 12:20:44 -07:00
2021-08-13 10:41:10 -07:00
mockApiServerOrigin: 'http://localhost:6060',
2021-08-13 17:22:28 -07:00
dashboardDomain: domain.domain,
dashboardFqdn: `my.${domain.domain}`,
2021-06-03 12:20:44 -07:00
2021-08-13 10:41:10 -07:00
app,
2022-11-23 14:36:57 +01:00
proxyApp,
2021-08-13 10:41:10 -07:00
admin,
auditSource,
2021-08-13 17:22:28 -07:00
domain, // the domain object
2021-08-13 15:49:59 -07:00
manifest,
2021-08-13 10:41:10 -07:00
user,
appstoreToken: 'atoken',
serverUrl: `http://localhost:${constants.PORT}`,
2017-09-29 09:56:01 -07:00
};
function createTree(root, obj) {
2022-11-06 11:48:56 +01:00
fs.rmSync(root, { recursive: true, force: true });
fs.mkdirSync(root, { recursive: true });
2017-09-29 09:56:01 -07:00
function createSubTree(tree, curpath) {
2022-04-14 20:30:00 -05:00
for (const key in tree) {
2017-09-29 09:56:01 -07:00
if (typeof tree[key] === 'string') {
if (key.startsWith('link:')) {
fs.symlinkSync(tree[key], path.join(curpath, key.slice(5)));
} else {
fs.writeFileSync(path.join(curpath, key), tree[key], 'utf8');
}
} else {
fs.mkdirSync(path.join(curpath, key));
createSubTree(tree[key], path.join(curpath, key));
}
}
}
createSubTree(obj, root);
}
2021-08-20 09:19:44 -07:00
async function databaseSetup() {
2021-06-03 12:20:44 -07:00
nock.cleanAll();
2021-08-20 09:19:44 -07:00
await database.initialize();
await database._clear();
2023-08-04 15:34:38 +05:30
await appstore._setApiServerOrigin(exports.mockApiServerOrigin);
await dashboard._setLocation(constants.DASHBOARD_SUBDOMAIN, exports.dashboardDomain);
2021-08-13 17:22:28 -07:00
}
2021-08-20 09:19:44 -07:00
async function domainSetup() {
2021-08-13 17:22:28 -07:00
nock.cleanAll();
2021-08-20 09:19:44 -07:00
await databaseSetup();
await mailServer.setLocation(constants.DASHBOARD_SUBDOMAIN, domain.domain); // default mail location. do this before we add the domain for upserting mail DNS
2021-08-20 09:19:44 -07:00
await domains.add(domain.domain, domain, auditSource);
2021-07-19 12:43:30 -07:00
}
2022-11-22 21:38:22 +01:00
async function setup() {
await fs.promises.rm(paths.DISK_USAGE_CACHE_FILE, { force: true });
2022-11-22 21:38:22 +01:00
await domainSetup();
const ownerId = await users.createOwner(admin.email, admin.username, admin.password, admin.displayName, auditSource);
admin.id = ownerId;
await apps.add(app.id, app.appStoreId, app.manifest, app.subdomain, app.domain, app.portBindings, app);
await settings._set(settings.APPSTORE_API_TOKEN_KEY, exports.appstoreToken); // appstore token
const userId = await users.add(user.email, user, auditSource);
user.id = userId;
await tasks.stopAllTasks();
remove global lock Currently, the update/apptask/fullbackup/platformstart take a global lock and cannot run in parallel. This causes situations where when a user tries to trigger an apptask, it says "waiting for backup to finish..." etc The solution is to let them run in parallel. We need a lock at the app level as app operations running in parallel would be bad (tm). In addition, the update task needs a lock just for the update part. We also need multi-process locks. Running tasks as processes is core to our "kill" strategy. Various inter process locks were explored: * node's IPC mechanism with process.send(). But this only works for direct node.js children. taskworker is run via sudo and the IPC does not work. * File lock using O_EXCL. Basic ideas to create lock files. While file creation can be done atomically, it becomes complicated to clean up lock files when the tasks crash. We need a way to know what locks were held by the crashing task. flock and friends are not built-into node.js * sqlite/redis were options but introduce additional deps * Settled on MySQL based locking. Initial plan was to have row locks or table locks. Each row is a kind of lock. While implementing, it was found that we need many types of locks (and not just update lock and app locks). For example, we need locks for each task type, so that only one task type is active at a time. * Instead of rows, we can just lock table and have a json blob in it. This hit a road block that LOCK TABLE is per session and our db layer cannot handle this easily! i.e when issing two db.query() it might use two different connections from the pool. We have to expose the connection, release connection etc. * Next idea was atomic blob update of the blob checking if old blob was same. This approach, was finally refined into a version field. Phew!
2024-12-07 14:35:45 +01:00
await locks.releaseAll();
2021-06-03 12:20:44 -07:00
}
2021-08-20 09:19:44 -07:00
async function cleanup() {
2021-06-03 12:20:44 -07:00
nock.cleanAll();
mailer._mailQueue = [];
await cron.stopJobs();
2021-08-20 09:19:44 -07:00
await database._clear();
await database.uninitialize();
2021-06-03 12:20:44 -07:00
}
2021-07-19 12:43:30 -07:00
2021-08-13 14:43:08 -07:00
function clearMailQueue() {
mailer._mailQueue = [];
}
async function checkMails(number) {
2023-05-14 10:53:50 +02:00
await timers.setTimeout(1000);
2021-08-13 14:43:08 -07:00
expect(mailer._mailQueue.length).to.equal(number);
const emails = mailer._mailQueue;
2021-08-13 14:43:08 -07:00
clearMailQueue();
// return for further investigation
return emails;
2021-08-13 14:43:08 -07:00
}