'use strict'; exports = module.exports = { testRegistryConfig, setRegistryConfig, injectPrivateFields, removePrivateFields, ping, info, downloadImage, createContainer, startContainer, restartContainer, stopContainer, stopContainerByName: stopContainer, stopContainers, deleteContainer, deleteImage, deleteContainers, createSubcontainer, getContainerIdByIp, inspect, getContainerIp, inspectByName: inspect, execContainer, getEvents, memoryUsage, createVolume, removeVolume, clearVolume, update }; const apps = require('./apps.js'), async = require('async'), assert = require('assert'), BoxError = require('./boxerror.js'), child_process = require('child_process'), constants = require('./constants.js'), debug = require('debug')('box:docker'), Docker = require('dockerode'), path = require('path'), reverseProxy = require('./reverseproxy.js'), services = require('./services.js'), settings = require('./settings.js'), shell = require('./shell.js'), safe = require('safetydance'), system = require('./system.js'), volumes = require('./volumes.js'), _ = require('underscore'); const CLEARVOLUME_CMD = path.join(__dirname, 'scripts/clearvolume.sh'), MKDIRVOLUME_CMD = path.join(__dirname, 'scripts/mkdirvolume.sh'); const DOCKER_SOCKET_PATH = '/var/run/docker.sock'; const gConnection = new Docker({ socketPath: DOCKER_SOCKET_PATH }); function testRegistryConfig(config, callback) { assert.strictEqual(typeof config, 'object'); assert.strictEqual(typeof callback, 'function'); if (config.provider === 'noop') return callback(); gConnection.checkAuth(config, function (error /*, data */) { // this returns a 500 even for auth errors if (error) return callback(new BoxError(BoxError.BAD_FIELD, error, { field: 'serverAddress' })); callback(); }); } function injectPrivateFields(newConfig, currentConfig) { if (newConfig.password === constants.SECRET_PLACEHOLDER) newConfig.password = currentConfig.password; } function removePrivateFields(registryConfig) { assert.strictEqual(typeof registryConfig, 'object'); if (registryConfig.password) registryConfig.password = constants.SECRET_PLACEHOLDER; return registryConfig; } function setRegistryConfig(config, callback) { assert.strictEqual(typeof config, 'object'); assert.strictEqual(typeof callback, 'function'); const isLogin = !!config.password; // currently, auth info is not stashed in the db but maybe it should for restore to work? const cmd = isLogin ? `docker login ${config.serverAddress} --username ${config.username} --password ${config.password}` : `docker logout ${config.serverAddress}`; child_process.exec(cmd, { }, function (error /*, stdout, stderr */) { if (error) return callback(new BoxError(BoxError.ACCESS_DENIED, error.message)); callback(); }); } function ping(callback) { assert.strictEqual(typeof callback, 'function'); // do not let the request linger const connection = new Docker({ socketPath: DOCKER_SOCKET_PATH, timeout: 1000 }); connection.ping(function (error, result) { if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error)); if (Buffer.isBuffer(result) && result.toString('utf8') === 'OK') return callback(null); // sometimes it returns buffer if (result === 'OK') return callback(null); callback(new BoxError(BoxError.DOCKER_ERROR, 'Unable to ping the docker daemon')); }); } function getRegistryConfig(image, callback) { // https://github.com/docker/distribution/blob/release/2.7/reference/normalize.go#L62 const parts = image.split('/'); if (parts.length === 1 || (parts[0].match(/[.:]/) === null)) return callback(null, null); // public docker registry settings.getRegistryConfig(function (error, registryConfig) { if (error) return callback(error); // https://github.com/apocas/dockerode#pull-from-private-repos const auth = { username: registryConfig.username, password: registryConfig.password, auth: registryConfig.auth || '', // the auth token at login time email: registryConfig.email || '', serveraddress: registryConfig.serverAddress }; callback(null, auth); }); } function pullImage(manifest, callback) { getRegistryConfig(manifest.dockerImage, function (error, config) { if (error) return callback(error); debug(`pullImage: will pull ${manifest.dockerImage}. auth: ${config ? 'yes' : 'no'}`); gConnection.pull(manifest.dockerImage, { authconfig: config }, function (error, stream) { if (error && error.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND, `Unable to pull image ${manifest.dockerImage}. message: ${error.message} statusCode: ${error.statusCode}`)); if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, `Unable to pull image ${manifest.dockerImage}. Please check the network or if the image needs authentication. statusCode: ${error.statusCode}`)); // https://github.com/dotcloud/docker/issues/1074 says each status message // is emitted as a chunk stream.on('data', function (chunk) { var data = safe.JSON.parse(chunk) || { }; debug('pullImage: %j', data); // The data.status here is useless because this is per layer as opposed to per image if (!data.status && data.error) { debug('pullImage error %s: %s', manifest.dockerImage, data.errorDetail.message); } }); stream.on('end', function () { debug('downloaded image %s', manifest.dockerImage); callback(null); }); stream.on('error', function (error) { debug('error pulling image %s: %j', manifest.dockerImage, error); callback(new BoxError(BoxError.DOCKER_ERROR, error.message)); }); }); }); } function downloadImage(manifest, callback) { assert.strictEqual(typeof manifest, 'object'); assert.strictEqual(typeof callback, 'function'); debug('downloadImage %s', manifest.dockerImage); const image = gConnection.getImage(manifest.dockerImage); image.inspect(function (error, result) { if (!error && result) return callback(null); // image is already present locally let attempt = 1; async.retry({ times: 10, interval: 5000, errorFilter: e => e.reason !== BoxError.NOT_FOUND }, function (retryCallback) { debug('Downloading image %s. attempt: %s', manifest.dockerImage, attempt++); pullImage(manifest, retryCallback); }, callback); }); } async function getVolumeMounts(app) { assert.strictEqual(typeof app, 'object'); let mounts = []; if (app.mounts.length === 0) return []; const result = await volumes.list(); let volumesById = {}; result.forEach(r => volumesById[r.id] = r); for (const mount of app.mounts) { const volume = volumesById[mount.volumeId]; mounts.push({ Source: volume.hostPath, Target: `/media/${volume.name}`, Type: 'bind', ReadOnly: mount.readOnly }); } return mounts; } function getAddonMounts(app, callback) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof callback, 'function'); let mounts = []; const addons = app.manifest.addons; if (!addons) return callback(null, mounts); async.eachSeries(Object.keys(addons), async function (addon) { switch (addon) { case 'localstorage': mounts.push({ Target: '/app/data', Source: `${app.id}-localstorage`, Type: 'volume', ReadOnly: false }); return; case 'tls': { const bundle = await reverseProxy.getCertificatePath(app.fqdn, app.domain); mounts.push({ Target: '/etc/certs/tls_cert.pem', Source: bundle.certFilePath, Type: 'bind', ReadOnly: true }); mounts.push({ Target: '/etc/certs/tls_key.pem', Source: bundle.keyFilePath, Type: 'bind', ReadOnly: true }); return; } default: return; } }, function (error) { callback(error, mounts); }); } async function getMounts(app, callback) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof callback, 'function'); const [error, volumeMounts] = await safe(getVolumeMounts(app)); if (error) return callback(error); getAddonMounts(app, function (error, addonMounts) { if (error) return callback(error); callback(null, volumeMounts.concat(addonMounts)); }); } function getAddresses() { const deviceLinks = safe.fs.readdirSync('/sys/class/net'); // https://man7.org/linux/man-pages/man5/sysfs.5.html if (!deviceLinks) return []; const devices = deviceLinks.map(d => { return { name: d, link: safe.fs.readlinkSync(`/sys/class/net/${d}`) }; }); const physicalDevices = devices.filter(d => d.link && !d.link.includes('virtual')); const addresses = []; for (const phy of physicalDevices) { const result = safe.JSON.parse(safe.child_process.execSync(`ip -f inet -j addr show ${phy.name}`, { encoding: 'utf8' })); const address = safe.query(result, '[0].addr_info[0].local'); if (address) addresses.push(address); } return addresses; } function createSubcontainer(app, name, cmd, options, callback) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof name, 'string'); assert(!cmd || Array.isArray(cmd)); assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof callback, 'function'); let isAppContainer = !cmd; // non app-containers are like scheduler var manifest = app.manifest; var exposedPorts = {}, dockerPortBindings = { }; var domain = app.fqdn; const envPrefix = manifest.manifestVersion <= 1 ? '' : 'CLOUDRON_'; let stdEnv = [ 'CLOUDRON=1', 'CLOUDRON_PROXY_IP=172.18.0.1', `CLOUDRON_APP_HOSTNAME=${app.id}`, `${envPrefix}WEBADMIN_ORIGIN=${settings.dashboardOrigin()}`, `${envPrefix}API_ORIGIN=${settings.dashboardOrigin()}`, `${envPrefix}APP_ORIGIN=https://${domain}`, `${envPrefix}APP_DOMAIN=${domain}` ]; var portEnv = []; for (let portName in app.portBindings) { const hostPort = app.portBindings[portName]; const portType = (manifest.tcpPorts && portName in manifest.tcpPorts) ? 'tcp' : 'udp'; const ports = portType == 'tcp' ? manifest.tcpPorts : manifest.udpPorts; var containerPort = ports[portName].containerPort || hostPort; // docker portBindings requires ports to be exposed exposedPorts[`${containerPort}/${portType}`] = {}; portEnv.push(`${portName}=${hostPort}`); const hostIps = hostPort === 53 ? getAddresses() : [ '0.0.0.0' ]; // port 53 is special because it is possibly taken by systemd-resolved dockerPortBindings[`${containerPort}/${portType}`] = hostIps.map(hip => { return { HostIp: hip, HostPort: hostPort + '' }; }); } let appEnv = []; Object.keys(app.env).forEach(function (name) { appEnv.push(`${name}=${app.env[name]}`); }); let memoryLimit = apps.getMemoryLimit(app); // give scheduler tasks twice the memory limit since background jobs take more memory // if required, we can make this a manifest and runtime argument later if (!isAppContainer) memoryLimit *= 2; services.getEnvironment(app, function (error, addonEnv) { if (error) return callback(error); getMounts(app, function (error, mounts) { if (error) return callback(error); let containerOptions = { name: name, // for referencing containers Tty: isAppContainer, Image: app.manifest.dockerImage, Cmd: (isAppContainer && app.debugMode && app.debugMode.cmd) ? app.debugMode.cmd : cmd, Env: stdEnv.concat(addonEnv).concat(portEnv).concat(appEnv), ExposedPorts: isAppContainer ? exposedPorts : { }, Volumes: { // see also ReadonlyRootfs '/tmp': {}, '/run': {} }, Labels: { 'fqdn': app.fqdn, 'appId': app.id, 'isSubcontainer': String(!isAppContainer), 'isCloudronManaged': String(true) }, HostConfig: { Mounts: mounts, LogConfig: { Type: 'syslog', Config: { 'tag': app.id, 'syslog-address': 'udp://127.0.0.1:2514', // see apps.js:validatePortBindings() 'syslog-format': 'rfc5424' } }, Memory: system.getMemoryAllocation(memoryLimit), MemorySwap: memoryLimit, // Memory + Swap PortBindings: isAppContainer ? dockerPortBindings : { }, PublishAllPorts: false, ReadonlyRootfs: app.debugMode ? !!app.debugMode.readonlyRootfs : true, RestartPolicy: { 'Name': isAppContainer ? 'unless-stopped' : 'no', 'MaximumRetryCount': 0 }, CpuShares: app.cpuShares, VolumesFrom: isAppContainer ? null : [ app.containerId + ':rw' ], SecurityOpt: [ 'apparmor=docker-cloudron-app' ], CapAdd: [], CapDrop: [] } }; // do no set hostname of containers to location as it might conflict with addons names. for example, an app installed in mail // location may not reach mail container anymore by DNS. We cannot set hostname to fqdn either as that sets up the dns // name to look up the internal docker ip. this makes curl from within container fail // Note that Hostname has no effect on DNS. We have to use the --net-alias for dns. // Hostname cannot be set with container NetworkMode. Subcontainers run is the network space of the app container // This is done to prevent lots of up/down events and iptables locking if (isAppContainer) { containerOptions.Hostname = app.id; containerOptions.HostConfig.NetworkMode = 'cloudron'; // user defined bridge network containerOptions.HostConfig.Dns = ['172.18.0.1']; // use internal dns containerOptions.HostConfig.DnsSearch = ['.']; // use internal dns containerOptions.NetworkingConfig = { EndpointsConfig: { cloudron: { IPAMConfig: { IPv4Address: app.containerIp }, Aliases: [ name ] // adds hostname entry with container name } } }; } else { containerOptions.HostConfig.NetworkMode = `container:${app.containerId}`; // scheduler containers must have same IP as app for various addon auth } var capabilities = manifest.capabilities || []; // https://docs-stage.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities if (capabilities.includes('net_admin')) containerOptions.HostConfig.CapAdd.push('NET_ADMIN', 'NET_RAW'); if (capabilities.includes('mlock')) containerOptions.HostConfig.CapAdd.push('IPC_LOCK'); // mlock prevents swapping if (!capabilities.includes('ping')) containerOptions.HostConfig.CapDrop.push('NET_RAW'); // NET_RAW is included by default by Docker if (capabilities.includes('vaapi') && safe.fs.existsSync('/dev/dri')) { containerOptions.HostConfig.Devices = [ { PathOnHost: '/dev/dri', PathInContainer: '/dev/dri', CgroupPermissions: 'rwm' } ]; } containerOptions = _.extend(containerOptions, options); gConnection.createContainer(containerOptions, function (error, container) { if (error && error.statusCode === 409) return callback(new BoxError(BoxError.ALREADY_EXISTS, error)); if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error)); callback(null, container); }); }); }); } function createContainer(app, callback) { createSubcontainer(app, app.id /* name */, null /* cmd */, { } /* options */, callback); } function startContainer(containerId, callback) { assert.strictEqual(typeof containerId, 'string'); assert.strictEqual(typeof callback, 'function'); var container = gConnection.getContainer(containerId); container.start(function (error) { if (error && error.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND)); if (error && error.statusCode === 400) return callback(new BoxError(BoxError.BAD_FIELD, error)); // e.g start.sh is not executable if (error && error.statusCode !== 304) return callback(new BoxError(BoxError.DOCKER_ERROR, error)); // 304 means already started return callback(null); }); } function restartContainer(containerId, callback) { assert.strictEqual(typeof containerId, 'string'); assert.strictEqual(typeof callback, 'function'); var container = gConnection.getContainer(containerId); container.restart(function (error) { if (error && error.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND)); if (error && error.statusCode === 400) return callback(new BoxError(BoxError.BAD_FIELD, error)); // e.g start.sh is not executable if (error && error.statusCode !== 204) return callback(new BoxError(BoxError.DOCKER_ERROR, error)); return callback(null); }); } function stopContainer(containerId, callback) { assert(!containerId || typeof containerId === 'string'); assert.strictEqual(typeof callback, 'function'); if (!containerId) { debug('No previous container to stop'); return callback(); } var container = gConnection.getContainer(containerId); var options = { t: 10 // wait for 10 seconds before killing it }; container.stop(options, function (error) { if (error && (error.statusCode !== 304 && error.statusCode !== 404)) return callback(new BoxError(BoxError.DOCKER_ERROR, 'Error stopping container:' + error.message)); container.wait(function (error/*, data */) { if (error && (error.statusCode !== 304 && error.statusCode !== 404)) return callback(new BoxError(BoxError.DOCKER_ERROR, 'Error waiting on container:' + error.message)); return callback(null); }); }); } function deleteContainer(containerId, callback) { // id can also be name assert(!containerId || typeof containerId === 'string'); assert.strictEqual(typeof callback, 'function'); if (containerId === null) return callback(null); var container = gConnection.getContainer(containerId); var removeOptions = { force: true, // kill container if it's running v: true // removes volumes associated with the container (but not host mounts) }; container.remove(removeOptions, function (error) { if (error && error.statusCode === 404) return callback(null); if (error) { debug('Error removing container %s : %j', containerId, error); return callback(new BoxError(BoxError.DOCKER_ERROR, error)); } callback(null); }); } function deleteContainers(appId, options, callback) { assert.strictEqual(typeof appId, 'string'); assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof callback, 'function'); let labels = [ 'appId=' + appId ]; if (options.managedOnly) labels.push('isCloudronManaged=true'); gConnection.listContainers({ all: 1, filters: JSON.stringify({ label: labels }) }, function (error, containers) { if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error)); async.eachSeries(containers, function (container, iteratorDone) { deleteContainer(container.Id, iteratorDone); }, callback); }); } function stopContainers(appId, callback) { assert.strictEqual(typeof appId, 'string'); assert.strictEqual(typeof callback, 'function'); gConnection.listContainers({ all: 1, filters: JSON.stringify({ label: [ 'appId=' + appId ] }) }, function (error, containers) { if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error)); async.eachSeries(containers, function (container, iteratorDone) { stopContainer(container.Id, iteratorDone); }, callback); }); } function deleteImage(manifest, callback) { assert(!manifest || typeof manifest === 'object'); assert.strictEqual(typeof callback, 'function'); const dockerImage = manifest ? manifest.dockerImage : null; if (!dockerImage) return callback(null); if (dockerImage.includes('//')) return callback(null); // a common mistake is to paste a https:// as docker image. this results in a crash at runtime in dockerode module const removeOptions = { force: false, // might be shared with another instance of this app noprune: false // delete untagged parents }; // registry v1 used to pull down all *tags*. this meant that deleting image by tag was not enough (since that // just removes the tag). we used to remove the image by id. this is not required anymore because aliases are // not created anymore after https://github.com/docker/docker/pull/10571 gConnection.getImage(dockerImage).remove(removeOptions, function (error) { if (error && error.statusCode === 400) return callback(null); // invalid image format. this can happen if user installed with a bad --docker-image if (error && error.statusCode === 404) return callback(null); // not found if (error && error.statusCode === 409) return callback(null); // another container using the image if (error) { debug('Error removing image %s : %j', dockerImage, error); return callback(new BoxError(BoxError.DOCKER_ERROR, error)); } callback(null); }); } function getContainerIdByIp(ip, callback) { assert.strictEqual(typeof ip, 'string'); assert.strictEqual(typeof callback, 'function'); gConnection.getNetwork('cloudron').inspect(function (error, bridge) { if (error && error.statusCode === 404) return callback(new BoxError(BoxError.DOCKER_ERROR, 'Unable to find the cloudron network')); if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error)); var containerId; for (var id in bridge.Containers) { if (bridge.Containers[id].IPv4Address.indexOf(ip + '/16') === 0) { containerId = id; break; } } if (!containerId) return callback(new BoxError(BoxError.DOCKER_ERROR, 'No container with that ip')); callback(null, containerId); }); } function inspect(containerId, callback) { assert.strictEqual(typeof containerId, 'string'); assert.strictEqual(typeof callback, 'function'); var container = gConnection.getContainer(containerId); container.inspect(function (error, result) { if (error && error.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND, `Unable to find container ${containerId}`)); if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error)); callback(null, result); }); } function getContainerIp(containerId, callback) { assert.strictEqual(typeof containerId, 'string'); assert.strictEqual(typeof callback, 'function'); if (constants.TEST) return callback(null, '127.0.5.5'); inspect(containerId, function (error, result) { if (error) return callback(error); const ip = safe.query(result, 'NetworkSettings.Networks.cloudron.IPAddress', null); if (!ip) return callback(new BoxError(BoxError.DOCKER_ERROR, 'Error getting container IP')); callback(null, ip); }); } function execContainer(containerId, options, callback) { assert.strictEqual(typeof containerId, 'string'); assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof callback, 'function'); var container = gConnection.getContainer(containerId); container.exec(options.execOptions, function (error, exec) { if (error && error.statusCode === 409) return callback(new BoxError(BoxError.BAD_STATE, error.message)); // container restarting/not running if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error)); exec.start(options.startOptions, function(error, stream /* in hijacked mode, this is a net.socket */) { if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error)); if (options.rows && options.columns) { // there is a race where resizing too early results in a 404 "no such exec" // https://git.cloudron.io/cloudron/box/issues/549 setTimeout(function () { exec.resize({ h: options.rows, w: options.columns }, function (error) { if (error) debug('Error resizing console', error); }); }, 2000); } callback(null, stream); }); }); } function getEvents(options, callback) { assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof callback, 'function'); gConnection.getEvents(options, function (error, stream) { if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error)); callback(null, stream); }); } function memoryUsage(containerId, callback) { assert.strictEqual(typeof containerId, 'string'); assert.strictEqual(typeof callback, 'function'); var container = gConnection.getContainer(containerId); container.stats({ stream: false }, function (error, result) { if (error && error.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND)); if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error)); callback(null, result); }); } function createVolume(name, volumeDataDir, labels, callback) { assert.strictEqual(typeof name, 'string'); assert.strictEqual(typeof volumeDataDir, 'string'); assert.strictEqual(typeof labels, 'object'); assert.strictEqual(typeof callback, 'function'); const volumeOptions = { Name: name, Driver: 'local', DriverOpts: { // https://github.com/moby/moby/issues/19990#issuecomment-248955005 type: 'none', device: volumeDataDir, o: 'bind' }, Labels: labels }; // requires sudo because the path can be outside appsdata shell.sudo('createVolume', [ MKDIRVOLUME_CMD, volumeDataDir ], {}, function (error) { if (error) return callback(new BoxError(BoxError.FS_ERROR, `Error creating app data dir: ${error.message}`)); gConnection.createVolume(volumeOptions, function (error) { if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error)); callback(); }); }); } function clearVolume(name, options, callback) { assert.strictEqual(typeof name, 'string'); assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof callback, 'function'); let volume = gConnection.getVolume(name); volume.inspect(function (error, v) { if (error && error.statusCode === 404) return callback(); if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error)); const volumeDataDir = v.Options.device; shell.sudo('clearVolume', [ CLEARVOLUME_CMD, options.removeDirectory ? 'rmdir' : 'clear', volumeDataDir ], {}, function (error) { if (error) return callback(new BoxError(BoxError.FS_ERROR, error)); callback(); }); }); } // this only removes the volume and not the data function removeVolume(name, callback) { assert.strictEqual(typeof name, 'string'); assert.strictEqual(typeof callback, 'function'); let volume = gConnection.getVolume(name); volume.remove(function (error) { if (error && error.statusCode !== 404) return callback(new BoxError(BoxError.DOCKER_ERROR, `removeVolume: Error removing volume: ${error.message}`)); callback(); }); } function info(callback) { assert.strictEqual(typeof callback, 'function'); gConnection.info(function (error, result) { if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, 'Error connecting to docker')); callback(null, result); }); } function update(name, memory, memorySwap, callback) { assert.strictEqual(typeof name, 'string'); assert.strictEqual(typeof memory, 'number'); assert.strictEqual(typeof memorySwap, 'number'); assert.strictEqual(typeof callback, 'function'); const args = `update --memory ${memory} --memory-swap ${memorySwap} ${name}`.split(' '); // scale back db containers, if possible. this is retried because updating memory constraints can fail // with failed to write to memory.memsw.limit_in_bytes: write /sys/fs/cgroup/memory/docker/xx/memory.memsw.limit_in_bytes: device or resource busy async.retry({ times: 10, interval: 60 * 1000 }, function (retryCallback) { shell.spawn(`update(${name})`, '/usr/bin/docker', args, { }, retryCallback); }, callback); }