'use strict'; exports = module.exports = { setupAddons: setupAddons, teardownAddons: teardownAddons, backupAddons: backupAddons, restoreAddons: restoreAddons, getEnvironment: getEnvironment, getLinksSync: getLinksSync, getBindsSync: getBindsSync, getContainerNamesSync: getContainerNamesSync, // exported for testing _setupOauth: setupOauth, _teardownOauth: teardownOauth }; var appdb = require('./appdb.js'), assert = require('assert'), async = require('async'), clients = require('./clients.js'), config = require('./config.js'), ClientsError = clients.ClientsError, debug = require('debug')('box:addons'), docker = require('./docker.js'), dockerConnection = docker.connection, fs = require('fs'), generatePassword = require('password-generator'), hat = require('hat'), infra = require('./infra_version.js'), once = require('once'), path = require('path'), paths = require('./paths.js'), safe = require('safetydance'), shell = require('./shell.js'), util = require('util'); var NOOP = function (app, options, callback) { return callback(); }; // setup can be called multiple times for the same app (configure crash restart) and existing data must not be lost // teardown is destructive. app data stored with the addon is lost var KNOWN_ADDONS = { email: { setup: setupEmail, teardown: teardownEmail, backup: NOOP, restore: setupEmail }, ldap: { setup: setupLdap, teardown: teardownLdap, backup: NOOP, restore: setupLdap }, localstorage: { setup: NOOP, // docker creates the directory for us teardown: NOOP, backup: NOOP, // no backup because it's already inside app data restore: NOOP }, mongodb: { setup: setupMongoDb, teardown: teardownMongoDb, backup: backupMongoDb, restore: restoreMongoDb }, mysql: { setup: setupMySql, teardown: teardownMySql, backup: backupMySql, restore: restoreMySql, }, oauth: { setup: setupOauth, teardown: teardownOauth, backup: NOOP, restore: setupOauth }, postgresql: { setup: setupPostgreSql, teardown: teardownPostgreSql, backup: backupPostgreSql, restore: restorePostgreSql }, recvmail: { setup: setupRecvMail, teardown: teardownRecvMail, backup: NOOP, restore: setupRecvMail }, redis: { setup: setupRedis, teardown: teardownRedis, backup: backupRedis, restore: setupRedis // same thing }, sendmail: { setup: setupSendMail, teardown: teardownSendMail, backup: NOOP, restore: setupSendMail }, scheduler: { setup: NOOP, teardown: NOOP, backup: NOOP, restore: NOOP }, simpleauth: { setup: setupSimpleAuth, teardown: teardownSimpleAuth, backup: NOOP, restore: setupSimpleAuth }, _docker: { setup: NOOP, teardown: NOOP, backup: NOOP, restore: NOOP } }; var RMAPPDIR_CMD = path.join(__dirname, 'scripts/rmappdir.sh'); function debugApp(app, args) { assert(!app || typeof app === 'object'); var prefix = app ? (app.location || 'naked_domain') : '(no app)'; debug(prefix + ' ' + util.format.apply(util, Array.prototype.slice.call(arguments, 1))); } function setupAddons(app, addons, callback) { assert.strictEqual(typeof app, 'object'); assert(!addons || typeof addons === 'object'); assert.strictEqual(typeof callback, 'function'); if (!addons) return callback(null); debugApp(app, 'setupAddons: Settings up %j', Object.keys(addons)); async.eachSeries(Object.keys(addons), function iterator(addon, iteratorCallback) { if (!(addon in KNOWN_ADDONS)) return iteratorCallback(new Error('No such addon:' + addon)); debugApp(app, 'Setting up addon %s with options %j', addon, addons[addon]); KNOWN_ADDONS[addon].setup(app, addons[addon], iteratorCallback); }, callback); } function teardownAddons(app, addons, callback) { assert.strictEqual(typeof app, 'object'); assert(!addons || typeof addons === 'object'); assert.strictEqual(typeof callback, 'function'); if (!addons) return callback(null); debugApp(app, 'teardownAddons: Tearing down %j', Object.keys(addons)); async.eachSeries(Object.keys(addons), function iterator(addon, iteratorCallback) { if (!(addon in KNOWN_ADDONS)) return iteratorCallback(new Error('No such addon:' + addon)); debugApp(app, 'Tearing down addon %s with options %j', addon, addons[addon]); KNOWN_ADDONS[addon].teardown(app, addons[addon], iteratorCallback); }, callback); } function backupAddons(app, addons, callback) { assert.strictEqual(typeof app, 'object'); assert(!addons || typeof addons === 'object'); assert.strictEqual(typeof callback, 'function'); debugApp(app, 'backupAddons'); if (!addons) return callback(null); debugApp(app, 'backupAddons: Backing up %j', Object.keys(addons)); async.eachSeries(Object.keys(addons), function iterator (addon, iteratorCallback) { if (!(addon in KNOWN_ADDONS)) return iteratorCallback(new Error('No such addon:' + addon)); KNOWN_ADDONS[addon].backup(app, addons[addon], iteratorCallback); }, callback); } function restoreAddons(app, addons, callback) { assert.strictEqual(typeof app, 'object'); assert(!addons || typeof addons === 'object'); assert.strictEqual(typeof callback, 'function'); debugApp(app, 'restoreAddons'); if (!addons) return callback(null); debugApp(app, 'restoreAddons: restoring %j', Object.keys(addons)); async.eachSeries(Object.keys(addons), function iterator (addon, iteratorCallback) { if (!(addon in KNOWN_ADDONS)) return iteratorCallback(new Error('No such addon:' + addon)); KNOWN_ADDONS[addon].restore(app, addons[addon], iteratorCallback); }, callback); } function getEnvironment(app, callback) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof callback, 'function'); appdb.getAddonConfigByAppId(app.id, callback); } function getLinksSync(app, addons) { assert.strictEqual(typeof app, 'object'); assert(!addons || typeof addons === 'object'); var links = [ ]; if (!addons) return links; var addMail = false; for (var addon in addons) { switch (addon) { case 'mysql': links.push('mysql:mysql'); break; case 'postgresql': links.push('postgresql:postgresql'); break; case 'sendmail': addMail = true; break; case 'recvmail': addMail = true; break; case 'email': addMail = true; break; case 'redis': links.push('redis-' + app.id + ':redis-' + app.id); break; case 'mongodb': links.push('mongodb:mongodb'); break; default: break; } } if (addMail) links.push('mail:mail'); return links; } function getBindsSync(app, addons) { assert.strictEqual(typeof app, 'object'); assert(!addons || typeof addons === 'object'); var binds = [ ]; if (!addons) return binds; for (var addon in addons) { switch (addon) { case '_docker': binds.push('/var/run/docker.sock:/var/run/docker.sock:rw'); break; case 'localstorage': binds.push(path.join(paths.DATA_DIR, app.id, 'data') + ':/app/data:rw'); break; default: break; } } return binds; } function getContainerNamesSync(app, addons) { assert.strictEqual(typeof app, 'object'); assert(!addons || typeof addons === 'object'); var names = [ ]; if (!addons) return names; for (var addon in addons) { switch (addon) { case 'scheduler': // names here depend on how scheduler.js creates containers names = names.concat(Object.keys(addons.scheduler).map(function (taskName) { return app.id + '-' + taskName; })); break; default: break; } } return names; } function setupOauth(app, options, callback) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof callback, 'function'); var appId = app.id; var redirectURI = 'https://' + config.appFqdn(app.location); var scope = 'profile'; clients.delByAppIdAndType(appId, clients.TYPE_OAUTH, function (error) { // remove existing creds if (error && error.reason !== ClientsError.NOT_FOUND) return callback(error); clients.add(appId, clients.TYPE_OAUTH, redirectURI, scope, function (error, result) { if (error) return callback(error); var env = [ 'OAUTH_CLIENT_ID=' + result.id, 'OAUTH_CLIENT_SECRET=' + result.clientSecret, 'OAUTH_ORIGIN=' + config.adminOrigin() ]; debugApp(app, 'Setting oauth addon config to %j', env); appdb.setAddonConfig(appId, 'oauth', env, callback); }); }); } function teardownOauth(app, options, callback) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof callback, 'function'); debugApp(app, 'teardownOauth'); clients.delByAppIdAndType(app.id, clients.TYPE_OAUTH, function (error) { if (error && error.reason !== ClientsError.NOT_FOUND) console.error(error); appdb.unsetAddonConfig(app.id, 'oauth', callback); }); } function setupSimpleAuth(app, options, callback) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof callback, 'function'); var appId = app.id; var scope = 'profile'; clients.delByAppIdAndType(app.id, clients.TYPE_SIMPLE_AUTH, function (error) { // remove existing creds if (error && error.reason !== ClientsError.NOT_FOUND) return callback(error); clients.add(appId, clients.TYPE_SIMPLE_AUTH, '', scope, function (error, result) { if (error) return callback(error); var env = [ 'SIMPLE_AUTH_SERVER=172.17.0.1', 'SIMPLE_AUTH_PORT=' + config.get('simpleAuthPort'), 'SIMPLE_AUTH_URL=http://172.17.0.1:' + config.get('simpleAuthPort'), // obsolete, remove 'SIMPLE_AUTH_ORIGIN=http://172.17.0.1:' + config.get('simpleAuthPort'), 'SIMPLE_AUTH_CLIENT_ID=' + result.id ]; debugApp(app, 'Setting simple auth addon config to %j', env); appdb.setAddonConfig(appId, 'simpleauth', env, callback); }); }); } function teardownSimpleAuth(app, options, callback) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof callback, 'function'); debugApp(app, 'teardownSimpleAuth'); clients.delByAppIdAndType(app.id, clients.TYPE_SIMPLE_AUTH, function (error) { if (error && error.reason !== ClientsError.NOT_FOUND) console.error(error); appdb.unsetAddonConfig(app.id, 'simpleauth', callback); }); } function setupEmail(app, options, callback) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof callback, 'function'); // note that "external" access info can be derived from MAIL_DOMAIN (since it's part of user documentation) var env = [ 'MAIL_SMTP_SERVER=mail', 'MAIL_SMTP_PORT=2525', 'MAIL_IMAP_SERVER=mail', 'MAIL_IMAP_PORT=9993', 'MAIL_SIEVE_SERVER=mail', 'MAIL_SIEVE_PORT=4190', 'MAIL_DOMAIN=' + config.fqdn() ]; debugApp(app, 'Setting up Email'); appdb.setAddonConfig(app.id, 'email', env, callback); } function teardownEmail(app, options, callback) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof callback, 'function'); debugApp(app, 'Tearing down Email'); appdb.unsetAddonConfig(app.id, 'email', callback); } function setupLdap(app, options, callback) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof callback, 'function'); var env = [ 'LDAP_SERVER=172.17.0.1', 'LDAP_PORT=' + config.get('ldapPort'), 'LDAP_URL=ldap://172.17.0.1:' + config.get('ldapPort'), 'LDAP_USERS_BASE_DN=ou=users,dc=cloudron', 'LDAP_GROUPS_BASE_DN=ou=groups,dc=cloudron', 'LDAP_BIND_DN=cn='+ app.id + ',ou=apps,dc=cloudron', 'LDAP_BIND_PASSWORD=' + hat(256) // this is ignored ]; debugApp(app, 'Setting up LDAP'); appdb.setAddonConfig(app.id, 'ldap', env, callback); } function teardownLdap(app, options, callback) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof callback, 'function'); debugApp(app, 'Tearing down LDAP'); appdb.unsetAddonConfig(app.id, 'ldap', callback); } function setupSendMail(app, options, callback) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof callback, 'function'); var from = (app.location ? app.location : app.manifest.title.replace(/[^a-zA-Z0-9]/, '')) + '.app'; var cmd = [ '/addons/mail/service.sh', 'add-send', from ]; docker.execContainer('mail', cmd, { bufferStdout: true }, function (error, stdout) { if (error) return callback(error); var env = stdout.toString('utf8').split('\n').slice(0, -1); // remove trailing newline debugApp(app, 'Setting sendmail addon config to %j', env); appdb.setAddonConfig(app.id, 'sendmail', env, callback); }); } function teardownSendMail(app, options, callback) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof callback, 'function'); debugApp(app, 'Tearing down sendmail'); var from = (app.location ? app.location : app.manifest.title.replace(/[^a-zA-Z0-9]/, '')) + '.app'; var cmd = [ '/addons/mail/service.sh', 'remove-send', from ]; debugApp(app, 'Tearing down sendmail'); docker.execContainer('mail', cmd, { }, function (error) { if (error) return callback(error); appdb.unsetAddonConfig(app.id, 'sendmail', callback); }); } function setupRecvMail(app, options, callback) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof callback, 'function'); debugApp(app, 'Setting up recvmail'); var to = (app.location ? app.location : app.manifest.title.replace(/[^a-zA-Z0-9]/, '')) + '.app'; var cmd = [ '/addons/mail/service.sh', 'add-recv', to ]; docker.execContainer('mail', cmd, { bufferStdout: true }, function (error, stdout) { if (error) return callback(error); var env = stdout.toString('utf8').split('\n').slice(0, -1); // remove trailing newline debugApp(app, 'Setting recvmail addon config to %j', env); appdb.setAddonConfig(app.id, 'recvmail', env, callback); }); } function teardownRecvMail(app, options, callback) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof callback, 'function'); var to = (app.location ? app.location : app.manifest.title.replace(/[^a-zA-Z0-9]/, '')) + '.app'; var cmd = [ '/addons/mail/service.sh', 'remove-recv', to ]; debugApp(app, 'Tearing down recvmail'); docker.execContainer('mail', cmd, { }, function (error) { if (error) return callback(error); appdb.unsetAddonConfig(app.id, 'recvmail', callback); }); } function setupMySql(app, options, callback) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof callback, 'function'); debugApp(app, 'Setting up mysql'); var cmd = [ '/addons/mysql/service.sh', options.multipleDatabases ? 'add-prefix' : 'add', app.id ]; docker.execContainer('mysql', cmd, { bufferStdout: true }, function (error, stdout) { if (error) return callback(error); var env = stdout.toString('utf8').split('\n').slice(0, -1); // remove trailing newline debugApp(app, 'Setting mysql addon config to %j', env); appdb.setAddonConfig(app.id, 'mysql', env, callback); }); } function teardownMySql(app, options, callback) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof callback, 'function'); var cmd = [ '/addons/mysql/service.sh', options.multipleDatabases ? 'remove-prefix' : 'remove', app.id ]; debugApp(app, 'Tearing down mysql'); docker.execContainer('mysql', cmd, { }, function (error) { if (error) return callback(error); appdb.unsetAddonConfig(app.id, 'mysql', callback); }); } function backupMySql(app, options, callback) { debugApp(app, 'Backing up mysql'); callback = once(callback); // ChildProcess exit may or may not be called after error var output = fs.createWriteStream(path.join(paths.DATA_DIR, app.id, 'mysqldump')); output.on('error', callback); var cmd = [ '/addons/mysql/service.sh', options.multipleDatabases ? 'backup-prefix' : 'backup', app.id ]; docker.execContainer('mysql', cmd, { stdout: output }, callback); } function restoreMySql(app, options, callback) { callback = once(callback); // ChildProcess exit may or may not be called after error setupMySql(app, options, function (error) { if (error) return callback(error); debugApp(app, 'restoreMySql'); var input = fs.createReadStream(path.join(paths.DATA_DIR, app.id, 'mysqldump')); input.on('error', callback); var cmd = [ '/addons/mysql/service.sh', options.multipleDatabases ? 'restore-prefix' : 'restore', app.id ]; docker.execContainer('mysql', cmd, { stdin: input }, callback); }); } function setupPostgreSql(app, options, callback) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof callback, 'function'); debugApp(app, 'Setting up postgresql'); var cmd = [ '/addons/postgresql/service.sh', 'add', app.id ]; docker.execContainer('postgresql', cmd, { bufferStdout: true }, function (error, stdout) { if (error) return callback(error); var env = stdout.toString('utf8').split('\n').slice(0, -1); // remove trailing newline debugApp(app, 'Setting postgresql addon config to %j', env); appdb.setAddonConfig(app.id, 'postgresql', env, callback); }); } function teardownPostgreSql(app, options, callback) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof callback, 'function'); var cmd = [ '/addons/postgresql/service.sh', 'remove', app.id ]; debugApp(app, 'Tearing down postgresql'); docker.execContainer('postgresql', cmd, { }, function (error) { if (error) return callback(error); appdb.unsetAddonConfig(app.id, 'postgresql', callback); }); } function backupPostgreSql(app, options, callback) { debugApp(app, 'Backing up postgresql'); callback = once(callback); // ChildProcess exit may or may not be called after error var output = fs.createWriteStream(path.join(paths.DATA_DIR, app.id, 'postgresqldump')); output.on('error', callback); var cmd = [ '/addons/postgresql/service.sh', 'backup', app.id ]; docker.execContainer('postgresql', cmd, { stdout: output }, callback); } function restorePostgreSql(app, options, callback) { callback = once(callback); setupPostgreSql(app, options, function (error) { if (error) return callback(error); debugApp(app, 'restorePostgreSql'); var input = fs.createReadStream(path.join(paths.DATA_DIR, app.id, 'postgresqldump')); input.on('error', callback); var cmd = [ '/addons/postgresql/service.sh', 'restore', app.id ]; docker.execContainer('postgresql', cmd, { stdin: input }, callback); }); } function setupMongoDb(app, options, callback) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof callback, 'function'); debugApp(app, 'Setting up mongodb'); var cmd = [ '/addons/mongodb/service.sh', 'add', app.id ]; docker.execContainer('mongodb', cmd, { bufferStdout: true }, function (error, stdout) { if (error) return callback(error); var env = stdout.toString('utf8').split('\n').slice(0, -1); // remove trailing newline debugApp(app, 'Setting mongodb addon config to %j', env); appdb.setAddonConfig(app.id, 'mongodb', env, callback); }); } function teardownMongoDb(app, options, callback) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof callback, 'function'); var cmd = [ '/addons/mongodb/service.sh', 'remove', app.id ]; debugApp(app, 'Tearing down mongodb'); docker.execContainer('mongodb', cmd, { }, function (error) { if (error) return callback(error); appdb.unsetAddonConfig(app.id, 'mongodb', callback); }); } function backupMongoDb(app, options, callback) { debugApp(app, 'Backing up mongodb'); callback = once(callback); // ChildProcess exit may or may not be called after error var output = fs.createWriteStream(path.join(paths.DATA_DIR, app.id, 'mongodbdump')); output.on('error', callback); var cmd = [ '/addons/mongodb/service.sh', 'backup', app.id ]; docker.execContainer('mongodb', cmd, { stdout: output }, callback); } function restoreMongoDb(app, options, callback) { callback = once(callback); // ChildProcess exit may or may not be called after error setupMongoDb(app, options, function (error) { if (error) return callback(error); debugApp(app, 'restoreMongoDb'); var input = fs.createReadStream(path.join(paths.DATA_DIR, app.id, 'mongodbdump')); input.on('error', callback); var cmd = [ '/addons/mongodb/service.sh', 'restore', app.id ]; docker.execContainer('mongodb', cmd, { stdin: input }, callback); }); } // Ensures that app's addon redis container is running. Can be called when named container already exists/running function setupRedis(app, options, callback) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof callback, 'function'); var redisPassword = generatePassword(64, false /* memorable */, /[\w\d_]/); // ensure no / in password for being sed friendly (and be uri friendly) var redisVarsFile = path.join(paths.ADDON_CONFIG_DIR, 'redis-' + app.id + '_vars.sh'); var redisDataDir = path.join(paths.DATA_DIR, app.id + '/redis'); if (!safe.fs.writeFileSync(redisVarsFile, 'REDIS_PASSWORD=' + redisPassword)) { return callback(new Error('Error writing redis config')); } if (!safe.fs.mkdirSync(redisDataDir) && safe.error.code !== 'EEXIST') return callback(new Error('Error creating redis data dir:' + safe.error)); var createOptions = { name: 'redis-' + app.id, Hostname: 'redis-' + app.location, Tty: true, Image: infra.images.redis.tag, Cmd: null, Volumes: { '/tmp': {}, '/run': {} }, VolumesFrom: [], HostConfig: { Binds: [ redisVarsFile + ':/etc/redis/redis_vars.sh:ro', redisDataDir + ':/var/lib/redis:rw' ], Memory: 1024 * 1024 * 75, // 100mb MemorySwap: 1024 * 1024 * 75 * 2, // 150mb PortBindings: { '6379/tcp': [{ HostPort: '0', HostIp: '127.0.0.1' }] }, ReadonlyRootfs: true, RestartPolicy: { 'Name': 'always', 'MaximumRetryCount': 0 } } }; var env = [ 'REDIS_URL=redis://redisuser:' + redisPassword + '@redis-' + app.id, 'REDIS_PASSWORD=' + redisPassword, 'REDIS_HOST=redis-' + app.id, 'REDIS_PORT=6379' ]; var redisContainer = dockerConnection.getContainer(createOptions.name); try { shell.execSync('stopRedis', 'docker stop --time=10 ' + createOptions.name + ' 2>/dev/null || true'); shell.execSync('stopRedis', 'docker rm --volumes ' + createOptions.name + ' 2>/dev/null || true'); } catch (e) { console.log('IGNORE EXCEPTIOn', e); } dockerConnection.createContainer(createOptions, function (error) { if (error && error.statusCode !== 409) return callback(error); // if not already created redisContainer.start(function (error) { if (error && error.statusCode !== 304) return callback(error); // if not already running appdb.setAddonConfig(app.id, 'redis', env, callback); }); }); } function teardownRedis(app, options, callback) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof callback, 'function'); var container = dockerConnection.getContainer('redis-' + app.id); var removeOptions = { force: true, // kill container if it's running v: true // removes volumes associated with the container }; container.remove(removeOptions, function (error) { if (error && error.statusCode !== 404) return callback(new Error('Error removing container:' + error)); safe.fs.unlinkSync(paths.ADDON_CONFIG_DIR, 'redis-' + app.id + '_vars.sh'); shell.sudo('teardownRedis', [ RMAPPDIR_CMD, app.id + '/redis' ], function (error, stdout, stderr) { if (error) return callback(new Error('Error removing redis data:' + error)); appdb.unsetAddonConfig(app.id, 'redis', callback); }); }); } function backupRedis(app, options, callback) { debugApp(app, 'Backing up redis'); var cmd = [ '/addons/redis/service.sh', 'backup' ]; // the redis dir is volume mounted docker.execContainer('redis-' + app.id, cmd, { }, callback); }