Files
cloudron-box/src/docker.js
Girish Ramakrishnan c0b0029935 statically allocate app container IPs
We removed httpPort with the assumption that docker allocated IPs
and kept them as long as the container is around. This turned out
to be not true because the IP changes on even container restart.

So we now allocate IPs statically. The iprange makes sure we don't
overlap with addons and other CI app or JupyterHub apps.

https://github.com/moby/moby/issues/6743
https://github.com/moby/moby/pull/19001
2020-11-20 16:19:59 -08:00

699 lines
27 KiB
JavaScript

'use strict';
exports = module.exports = {
testRegistryConfig,
setRegistryConfig,
injectPrivateFields,
removePrivateFields,
ping,
info,
downloadImage,
createContainer,
startContainer,
restartContainer,
stopContainer,
stopContainerByName: stopContainer,
stopContainers,
deleteContainer,
deleteImage,
deleteContainers,
createSubcontainer,
getContainerIdByIp,
inspect,
getContainerIp,
inspectByName: inspect,
execContainer,
getEvents,
memoryUsage,
createVolume,
removeVolume,
clearVolume
};
var addons = require('./addons.js'),
async = require('async'),
assert = require('assert'),
BoxError = require('./boxerror.js'),
child_process = require('child_process'),
constants = require('./constants.js'),
debug = require('debug')('box:docker'),
Docker = require('dockerode'),
os = require('os'),
path = require('path'),
settings = require('./settings.js'),
shell = require('./shell.js'),
safe = require('safetydance'),
util = require('util'),
volumes = require('./volumes.js'),
_ = require('underscore');
const CLEARVOLUME_CMD = path.join(__dirname, 'scripts/clearvolume.sh'),
MKDIRVOLUME_CMD = path.join(__dirname, 'scripts/mkdirvolume.sh');
const DOCKER_SOCKET_PATH = '/var/run/docker.sock';
const gConnection = new Docker({ socketPath: DOCKER_SOCKET_PATH });
function testRegistryConfig(auth, callback) {
assert.strictEqual(typeof auth, 'object');
assert.strictEqual(typeof callback, 'function');
gConnection.checkAuth(auth, function (error /*, data */) { // this returns a 500 even for auth errors
if (error) return callback(new BoxError(BoxError.BAD_FIELD, error, { field: 'serverAddress' }));
callback();
});
}
function injectPrivateFields(newConfig, currentConfig) {
if (newConfig.password === constants.SECRET_PLACEHOLDER) newConfig.password = currentConfig.password;
}
function removePrivateFields(registryConfig) {
assert.strictEqual(typeof registryConfig, 'object');
if (registryConfig.password) registryConfig.password = constants.SECRET_PLACEHOLDER;
return registryConfig;
}
function setRegistryConfig(auth, callback) {
assert.strictEqual(typeof auth, 'object');
assert.strictEqual(typeof callback, 'function');
const isLogin = !!auth.password;
// currently, auth info is not stashed in the db but maybe it should for restore to work?
const cmd = isLogin ? `docker login ${auth.serverAddress} --username ${auth.username} --password ${auth.password}` : `docker logout ${auth.serverAddress}`;
child_process.exec(cmd, { }, function (error /*, stdout, stderr */) {
if (error) return callback(new BoxError(BoxError.ACCESS_DENIED, error.message));
callback();
});
}
function ping(callback) {
assert.strictEqual(typeof callback, 'function');
// do not let the request linger
const connection = new Docker({ socketPath: DOCKER_SOCKET_PATH, timeout: 1000 });
connection.ping(function (error, result) {
if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error));
if (Buffer.isBuffer(result) && result.toString('utf8') === 'OK') return callback(null); // sometimes it returns buffer
if (result === 'OK') return callback(null);
callback(new BoxError(BoxError.DOCKER_ERROR, 'Unable to ping the docker daemon'));
});
}
function getRegistryConfig(image, callback) {
// https://github.com/docker/distribution/blob/release/2.7/reference/normalize.go#L62
const parts = image.split('/');
if (parts.length === 1 || (parts[0].match(/[.:]/) === null)) return callback(null, null); // public docker registry
settings.getRegistryConfig(function (error, registryConfig) {
if (error) return callback(error);
// https://github.com/apocas/dockerode#pull-from-private-repos
const auth = {
username: registryConfig.username,
password: registryConfig.password,
auth: registryConfig.auth || '', // the auth token at login time
email: registryConfig.email || '',
serveraddress: registryConfig.serverAddress
};
callback(null, auth);
});
}
function pullImage(manifest, callback) {
getRegistryConfig(manifest.dockerImage, function (error, authConfig) {
if (error) return callback(error);
debug(`pullImage: will pull ${manifest.dockerImage}. auth: ${authConfig ? 'yes' : 'no'}`);
gConnection.pull(manifest.dockerImage, { authconfig: authConfig }, function (error, stream) {
if (error && error.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND, `Unable to pull image ${manifest.dockerImage}. message: ${error.message} statusCode: ${error.statusCode}`));
if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, `Unable to pull image ${manifest.dockerImage}. Please check the network or if the image needs authentication. statusCode: ${error.statusCode}`));
// https://github.com/dotcloud/docker/issues/1074 says each status message
// is emitted as a chunk
stream.on('data', function (chunk) {
var data = safe.JSON.parse(chunk) || { };
debug('pullImage: %j', data);
// The data.status here is useless because this is per layer as opposed to per image
if (!data.status && data.error) {
debug('pullImage error %s: %s', manifest.dockerImage, data.errorDetail.message);
}
});
stream.on('end', function () {
debug('downloaded image %s', manifest.dockerImage);
callback(null);
});
stream.on('error', function (error) {
debug('error pulling image %s: %j', manifest.dockerImage, error);
callback(new BoxError(BoxError.DOCKER_ERROR, error.message));
});
});
});
}
function downloadImage(manifest, callback) {
assert.strictEqual(typeof manifest, 'object');
assert.strictEqual(typeof callback, 'function');
debug('downloadImage %s', manifest.dockerImage);
const image = gConnection.getImage(manifest.dockerImage);
image.inspect(function (error, result) {
if (!error && result) return callback(null); // image is already present locally
let attempt = 1;
async.retry({ times: 10, interval: 5000, errorFilter: e => e.reason !== BoxError.NOT_FOUND }, function (retryCallback) {
debug('Downloading image %s. attempt: %s', manifest.dockerImage, attempt++);
pullImage(manifest, retryCallback);
}, callback);
});
}
function getBinds(app, callback) {
assert.strictEqual(typeof app, 'object');
assert.strictEqual(typeof callback, 'function');
if (app.mounts.length === 0) return callback(null);
let binds = [];
volumes.list(function (error, result) {
if (error) return callback(error);
let volumesById = {};
result.forEach(r => volumesById[r.id] = r);
for (const mount of app.mounts) {
const volume = volumesById[mount.volumeId];
binds.push(`${volume.hostPath}:/media/${volume.name}:${mount.readOnly ? 'ro' : 'rw'}`);
}
callback(null, binds);
});
}
function getLowerUpIp() { // see getifaddrs and IFF_LOWER_UP and netdevice
const ni = os.networkInterfaces(); // { lo: [], eth0: [] }
for (const iname of Object.keys(ni)) {
if (iname === 'lo') continue;
for (const address of ni[iname]) {
if (!address.internal && address.family === 'IPv4') return address.address;
}
}
return null;
}
function createSubcontainer(app, name, cmd, options, callback) {
assert.strictEqual(typeof app, 'object');
assert.strictEqual(typeof name, 'string');
assert(!cmd || util.isArray(cmd));
assert.strictEqual(typeof options, 'object');
assert.strictEqual(typeof callback, 'function');
let isAppContainer = !cmd; // non app-containers are like scheduler
var manifest = app.manifest;
var exposedPorts = {}, dockerPortBindings = { };
var domain = app.fqdn;
const envPrefix = manifest.manifestVersion <= 1 ? '' : 'CLOUDRON_';
let stdEnv = [
'CLOUDRON=1',
'CLOUDRON_PROXY_IP=172.18.0.1',
`CLOUDRON_APP_HOSTNAME=${app.id}`,
`${envPrefix}WEBADMIN_ORIGIN=${settings.adminOrigin()}`,
`${envPrefix}API_ORIGIN=${settings.adminOrigin()}`,
`${envPrefix}APP_ORIGIN=https://${domain}`,
`${envPrefix}APP_DOMAIN=${domain}`
];
var portEnv = [];
for (let portName in app.portBindings) {
const hostPort = app.portBindings[portName];
const portType = (manifest.tcpPorts && portName in manifest.tcpPorts) ? 'tcp' : 'udp';
const ports = portType == 'tcp' ? manifest.tcpPorts : manifest.udpPorts;
var containerPort = ports[portName].containerPort || hostPort;
// docker portBindings requires ports to be exposed
exposedPorts[`${containerPort}/${portType}`] = {};
portEnv.push(`${portName}=${hostPort}`);
const hostIp = hostPort === 53 ? getLowerUpIp() : '0.0.0.0'; // port 53 is special because it is possibly taken by systemd-resolved
dockerPortBindings[`${containerPort}/${portType}`] = [ { HostIp: hostIp, HostPort: hostPort + '' } ];
}
let appEnv = [];
Object.keys(app.env).forEach(function (name) { appEnv.push(`${name}=${app.env[name]}`); });
// first check db record, then manifest
var memoryLimit = app.memoryLimit || manifest.memoryLimit || 0;
if (memoryLimit === -1) { // unrestricted
memoryLimit = 0;
} else if (memoryLimit === 0 || memoryLimit < constants.DEFAULT_MEMORY_LIMIT) { // ensure we never go below minimum (in case we change the default)
memoryLimit = constants.DEFAULT_MEMORY_LIMIT;
}
// give scheduler tasks twice the memory limit since background jobs take more memory
// if required, we can make this a manifest and runtime argument later
if (!isAppContainer) memoryLimit *= 2;
addons.getEnvironment(app, function (error, addonEnv) {
if (error) return callback(error);
getBinds(app, function (error, binds) {
if (error) return callback(error);
let containerOptions = {
name: name, // for referencing containers
Tty: isAppContainer,
Image: app.manifest.dockerImage,
Cmd: (isAppContainer && app.debugMode && app.debugMode.cmd) ? app.debugMode.cmd : cmd,
Env: stdEnv.concat(addonEnv).concat(portEnv).concat(appEnv),
ExposedPorts: isAppContainer ? exposedPorts : { },
Volumes: { // see also ReadonlyRootfs
'/tmp': {},
'/run': {}
},
Labels: {
'fqdn': app.fqdn,
'appId': app.id,
'isSubcontainer': String(!isAppContainer),
'isCloudronManaged': String(true)
},
HostConfig: {
Mounts: addons.getMountsSync(app, app.manifest.addons),
Binds: binds, // ideally, we have to use 'Mounts' but we have to create volumes then
LogConfig: {
Type: 'syslog',
Config: {
'tag': app.id,
'syslog-address': 'udp://127.0.0.1:2514', // see apps.js:validatePortBindings()
'syslog-format': 'rfc5424'
}
},
Memory: memoryLimit / 2,
MemorySwap: memoryLimit, // Memory + Swap
PortBindings: isAppContainer ? dockerPortBindings : { },
PublishAllPorts: false,
ReadonlyRootfs: app.debugMode ? !!app.debugMode.readonlyRootfs : true,
RestartPolicy: {
'Name': isAppContainer ? 'unless-stopped' : 'no',
'MaximumRetryCount': 0
},
CpuShares: app.cpuShares,
VolumesFrom: isAppContainer ? null : [ app.containerId + ':rw' ],
SecurityOpt: [ 'apparmor=docker-cloudron-app' ],
CapAdd: [],
CapDrop: []
}
};
// do no set hostname of containers to location as it might conflict with addons names. for example, an app installed in mail
// location may not reach mail container anymore by DNS. We cannot set hostname to fqdn either as that sets up the dns
// name to look up the internal docker ip. this makes curl from within container fail
// Note that Hostname has no effect on DNS. We have to use the --net-alias for dns.
// Hostname cannot be set with container NetworkMode. Subcontainers run is the network space of the app container
// This is done to prevent lots of up/down events and iptables locking
if (isAppContainer) {
containerOptions.Hostname = app.id;
containerOptions.HostConfig.NetworkMode = 'cloudron'; // user defined bridge network
containerOptions.NetworkingConfig = {
EndpointsConfig: {
cloudron: {
IPAMConfig: {
IPv4Address: app.containerIp
},
Aliases: [ name ] // adds hostname entry with container name
}
}
};
} else {
containerOptions.HostConfig.NetworkMode = `container:${app.containerId}`;
}
var capabilities = manifest.capabilities || [];
// https://docs-stage.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities
if (capabilities.includes('net_admin')) containerOptions.HostConfig.CapAdd.push('NET_ADMIN', 'NET_RAW');
if (capabilities.includes('mlock')) containerOptions.HostConfig.CapAdd.push('IPC_LOCK'); // mlock prevents swapping
if (!capabilities.includes('ping')) containerOptions.HostConfig.CapDrop.push('NET_RAW'); // NET_RAW is included by default by Docker
if (capabilities.includes('vaapi') && safe.fs.existsSync('/dev/dri')) {
containerOptions.HostConfig.Devices = [
{ PathOnHost: '/dev/dri', PathInContainer: '/dev/dri', CgroupPermissions: 'rwm' }
];
}
containerOptions = _.extend(containerOptions, options);
gConnection.createContainer(containerOptions, function (error, container) {
if (error && error.statusCode === 409) return callback(new BoxError(BoxError.ALREADY_EXISTS, error));
if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error));
callback(null, container);
});
});
});
}
function createContainer(app, callback) {
createSubcontainer(app, app.id /* name */, null /* cmd */, { } /* options */, callback);
}
function startContainer(containerId, callback) {
assert.strictEqual(typeof containerId, 'string');
assert.strictEqual(typeof callback, 'function');
var container = gConnection.getContainer(containerId);
container.start(function (error) {
if (error && error.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND));
if (error && error.statusCode === 400) return callback(new BoxError(BoxError.BAD_FIELD, error)); // e.g start.sh is not executable
if (error && error.statusCode !== 304) return callback(new BoxError(BoxError.DOCKER_ERROR, error)); // 304 means already started
return callback(null);
});
}
function restartContainer(containerId, callback) {
assert.strictEqual(typeof containerId, 'string');
assert.strictEqual(typeof callback, 'function');
var container = gConnection.getContainer(containerId);
container.restart(function (error) {
if (error && error.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND));
if (error && error.statusCode === 400) return callback(new BoxError(BoxError.BAD_FIELD, error)); // e.g start.sh is not executable
if (error && error.statusCode !== 204) return callback(new BoxError(BoxError.DOCKER_ERROR, error));
return callback(null);
});
}
function stopContainer(containerId, callback) {
assert(!containerId || typeof containerId === 'string');
assert.strictEqual(typeof callback, 'function');
if (!containerId) {
debug('No previous container to stop');
return callback();
}
var container = gConnection.getContainer(containerId);
var options = {
t: 10 // wait for 10 seconds before killing it
};
container.stop(options, function (error) {
if (error && (error.statusCode !== 304 && error.statusCode !== 404)) return callback(new BoxError(BoxError.DOCKER_ERROR, 'Error stopping container:' + error.message));
container.wait(function (error/*, data */) {
if (error && (error.statusCode !== 304 && error.statusCode !== 404)) return callback(new BoxError(BoxError.DOCKER_ERROR, 'Error waiting on container:' + error.message));
return callback(null);
});
});
}
function deleteContainer(containerId, callback) { // id can also be name
assert(!containerId || typeof containerId === 'string');
assert.strictEqual(typeof callback, 'function');
if (containerId === null) return callback(null);
var container = gConnection.getContainer(containerId);
var removeOptions = {
force: true, // kill container if it's running
v: true // removes volumes associated with the container (but not host mounts)
};
container.remove(removeOptions, function (error) {
if (error && error.statusCode === 404) return callback(null);
if (error) {
debug('Error removing container %s : %j', containerId, error);
return callback(new BoxError(BoxError.DOCKER_ERROR, error));
}
callback(null);
});
}
function deleteContainers(appId, options, callback) {
assert.strictEqual(typeof appId, 'string');
assert.strictEqual(typeof options, 'object');
assert.strictEqual(typeof callback, 'function');
let labels = [ 'appId=' + appId ];
if (options.managedOnly) labels.push('isCloudronManaged=true');
gConnection.listContainers({ all: 1, filters: JSON.stringify({ label: labels }) }, function (error, containers) {
if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error));
async.eachSeries(containers, function (container, iteratorDone) {
deleteContainer(container.Id, iteratorDone);
}, callback);
});
}
function stopContainers(appId, callback) {
assert.strictEqual(typeof appId, 'string');
assert.strictEqual(typeof callback, 'function');
gConnection.listContainers({ all: 1, filters: JSON.stringify({ label: [ 'appId=' + appId ] }) }, function (error, containers) {
if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error));
async.eachSeries(containers, function (container, iteratorDone) {
stopContainer(container.Id, iteratorDone);
}, callback);
});
}
function deleteImage(manifest, callback) {
assert(!manifest || typeof manifest === 'object');
assert.strictEqual(typeof callback, 'function');
var dockerImage = manifest ? manifest.dockerImage : null;
if (!dockerImage) return callback(null);
var removeOptions = {
force: false, // might be shared with another instance of this app
noprune: false // delete untagged parents
};
// registry v1 used to pull down all *tags*. this meant that deleting image by tag was not enough (since that
// just removes the tag). we used to remove the image by id. this is not required anymore because aliases are
// not created anymore after https://github.com/docker/docker/pull/10571
gConnection.getImage(dockerImage).remove(removeOptions, function (error) {
if (error && error.statusCode === 400) return callback(null); // invalid image format. this can happen if user installed with a bad --docker-image
if (error && error.statusCode === 404) return callback(null); // not found
if (error && error.statusCode === 409) return callback(null); // another container using the image
if (error) {
debug('Error removing image %s : %j', dockerImage, error);
return callback(new BoxError(BoxError.DOCKER_ERROR, error));
}
callback(null);
});
}
function getContainerIdByIp(ip, callback) {
assert.strictEqual(typeof ip, 'string');
assert.strictEqual(typeof callback, 'function');
gConnection.getNetwork('cloudron').inspect(function (error, bridge) {
if (error && error.statusCode === 404) return callback(new BoxError(BoxError.DOCKER_ERROR, 'Unable to find the cloudron network'));
if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error));
var containerId;
for (var id in bridge.Containers) {
if (bridge.Containers[id].IPv4Address.indexOf(ip + '/16') === 0) {
containerId = id;
break;
}
}
if (!containerId) return callback(new BoxError(BoxError.DOCKER_ERROR, 'No container with that ip'));
callback(null, containerId);
});
}
function inspect(containerId, callback) {
assert.strictEqual(typeof containerId, 'string');
assert.strictEqual(typeof callback, 'function');
var container = gConnection.getContainer(containerId);
container.inspect(function (error, result) {
if (error && error.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND, `Unable to find container ${containerId}`));
if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error));
callback(null, result);
});
}
function getContainerIp(containerId, callback) {
assert.strictEqual(typeof containerId, 'string');
assert.strictEqual(typeof callback, 'function');
if (constants.TEST) return callback(null, '127.0.5.5');
inspect(containerId, function (error, result) {
if (error) return callback(error);
const ip = safe.query(result, 'NetworkSettings.Networks.cloudron.IPAddress', null);
if (!ip) return callback(new BoxError(BoxError.DOCKER_ERROR, 'Error getting container IP'));
callback(null, ip);
});
}
function execContainer(containerId, options, callback) {
assert.strictEqual(typeof containerId, 'string');
assert.strictEqual(typeof options, 'object');
assert.strictEqual(typeof callback, 'function');
var container = gConnection.getContainer(containerId);
container.exec(options.execOptions, function (error, exec) {
if (error && error.statusCode === 409) return callback(new BoxError(BoxError.BAD_STATE, error.message)); // container restarting/not running
if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error));
exec.start(options.startOptions, function(error, stream /* in hijacked mode, this is a net.socket */) {
if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error));
if (options.rows && options.columns) {
// there is a race where resizing too early results in a 404 "no such exec"
// https://git.cloudron.io/cloudron/box/issues/549
setTimeout(function () {
exec.resize({ h: options.rows, w: options.columns }, function (error) { if (error) debug('Error resizing console', error); });
}, 2000);
}
callback(null, stream);
});
});
}
function getEvents(options, callback) {
assert.strictEqual(typeof options, 'object');
assert.strictEqual(typeof callback, 'function');
gConnection.getEvents(options, function (error, stream) {
if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error));
callback(null, stream);
});
}
function memoryUsage(containerId, callback) {
assert.strictEqual(typeof containerId, 'string');
assert.strictEqual(typeof callback, 'function');
var container = gConnection.getContainer(containerId);
container.stats({ stream: false }, function (error, result) {
if (error && error.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND));
if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error));
callback(null, result);
});
}
function createVolume(name, volumeDataDir, labels, callback) {
assert.strictEqual(typeof name, 'string');
assert.strictEqual(typeof volumeDataDir, 'string');
assert.strictEqual(typeof labels, 'object');
assert.strictEqual(typeof callback, 'function');
const volumeOptions = {
Name: name,
Driver: 'local',
DriverOpts: { // https://github.com/moby/moby/issues/19990#issuecomment-248955005
type: 'none',
device: volumeDataDir,
o: 'bind'
},
Labels: labels
};
// requires sudo because the path can be outside appsdata
shell.sudo('createVolume', [ MKDIRVOLUME_CMD, volumeDataDir ], {}, function (error) {
if (error) return callback(new BoxError(BoxError.FS_ERROR, `Error creating app data dir: ${error.message}`));
gConnection.createVolume(volumeOptions, function (error) {
if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error));
callback();
});
});
}
function clearVolume(name, options, callback) {
assert.strictEqual(typeof name, 'string');
assert.strictEqual(typeof options, 'object');
assert.strictEqual(typeof callback, 'function');
let volume = gConnection.getVolume(name);
volume.inspect(function (error, v) {
if (error && error.statusCode === 404) return callback();
if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error));
const volumeDataDir = v.Options.device;
shell.sudo('clearVolume', [ CLEARVOLUME_CMD, options.removeDirectory ? 'rmdir' : 'clear', volumeDataDir ], {}, function (error) {
if (error) return callback(new BoxError(BoxError.FS_ERROR, error));
callback();
});
});
}
// this only removes the volume and not the data
function removeVolume(name, callback) {
assert.strictEqual(typeof name, 'string');
assert.strictEqual(typeof callback, 'function');
let volume = gConnection.getVolume(name);
volume.remove(function (error) {
if (error && error.statusCode !== 404) return callback(new BoxError(BoxError.DOCKER_ERROR, `removeVolume: Error removing volume: ${error.message}`));
callback();
});
}
function info(callback) {
assert.strictEqual(typeof callback, 'function');
gConnection.info(function (error, result) {
if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, 'Error connecting to docker'));
callback(null, result);
});
}