Implement runtime state

This separation makes it possible for apphealthtask to go in parallel
and not worry about it the app being uninstalled.

Fixes #27
This commit is contained in:
Girish Ramakrishnan
2014-06-28 02:28:04 -07:00
parent 434693976f
commit 8f27c01cf4
5 changed files with 68 additions and 82 deletions
+7 -7
View File
@@ -31,13 +31,10 @@ exports = module.exports = {
ISTATE_DOWNLOADING_IMAGE: 'downloading_image',
ISTATE_DOWNLOADED_IMAGE: 'downloaded_image',
ISTATE_IMAGE_ERROR: 'image_error',
ISTATE_CREATING_CONTAINER: 'creating_container',
ISTATE_CREATED_CONTAINER: 'created_container',
ISTATE_STARTING_CONTAINER: 'starting_container',
ISTATE_STARTED_CONTAINER: 'started_container',
ISTATE_CONTAINER_ERROR: 'container_error',
ISTATE_CREATING_VOLUME: 'creating_volume',
ISTATE_CREATED_VOLUME: 'created_volume',
ISTATE_VOLUME_ERROR: 'volume_error',
@@ -46,9 +43,12 @@ exports = module.exports = {
ISTATE_REGISTERED_SUBDOMAIN: 'registered_subdomain',
ISTATE_SUBDOMAIN_ERROR: 'subdomain_error',
ISTATE_RUNNING: 'running',
ISTATE_EXITED: 'exited',
ISTATE_NOT_RESPONDING: 'not_responding'
ISTATE_INSTALLED: 'installed',
RSTATE_RUNNING: 'running',
RSTATE_ERROR: 'error',
RSTATE_EXITED: 'exited',
RSTATE_NOT_RESPONDING: 'not_responding'
};
var DatabaseError = require('./databaseerror'),
+22 -27
View File
@@ -5,6 +5,7 @@
var assert = require('assert'),
config = require('../config.js'),
database = require('./database.js'),
DatabaseError = require('./databaseerror'),
appdb = require('./appdb.js'),
async = require('async'),
Docker = require('dockerode'),
@@ -38,29 +39,36 @@ function initialize() {
});
}
function updateApp(app, values, callback) {
for (var value in values) {
app[value] = values[value];
}
appdb.update(app.id, values, callback);
}
// # TODO should probably poll from the outside network instead of the docker network?
// callback is called with error for fatal errors and not if health check failed
function checkAppHealth(app, callback) {
if (app.installationState !== appdb.ISTATE_INSTALLED) return callback(null);
var container = docker.getContainer(app.containerId),
manifest = JSON.parse(app.manifestJson); // this is guaranteed not to throw since it's already been verified in downloadManifest()
function updateApp(app, values, callback) {
for (var value in values) {
app[value] = values[value];
}
appdb.update(app.id, values, function (error) {
if (error && error.reason === DatabaseError.NOT_FOUND) { // app got uninstalled
return callback(null);
}
callback(error);
});
}
container.inspect(function (err, data) {
if (err || !data || !data.State) {
debug('Error inspecting container');
updateApp(app, { installationState: appdb.ISTATE_IMAGE_ERROR }, FATAL_CALLBACK);
return callback(err);
return updateApp(app, { runState: appdb.RSTATE_ERROR }, callback);
}
if (data.State.Running !== true) {
debug(app.id + ' has exited');
updateApp(app, { installationState: appdb.ISTATE_EXITED }, FATAL_CALLBACK);
return callback(null);
return updateApp(app, { runState: appdb.RSTATE_EXITED }, callback);
}
var healthCheckUrl = 'http://127.0.0.1:' + app.httpPort + manifest.health_check_url;
@@ -71,12 +79,10 @@ function checkAppHealth(app, callback) {
if (error || res.status !== 200) {
debug('Marking application as dead: ' + app.id);
updateApp(app, { installationState: appdb.ISTATE_NOT_RESPONDING }, FATAL_CALLBACK);
callback(null);
updateApp(app, { runState: appdb.RSTATE_NOT_RESPONDING }, callback);
} else {
debug('healthy app:' + app.id);
updateApp(app, { installationState: appdb.ISTATE_RUNNING }, FATAL_CALLBACK);
callback(null);
updateApp(app, { runState: appdb.RSTATE_RUNNING }, callback);
}
});
});
@@ -86,18 +92,7 @@ function processApps(callback) {
appdb.getAll(function (error, apps) {
if (error) return callback(error);
async.each(apps, function (app, done) {
switch (app.installationState) {
case appdb.ISTATE_RUNNING:
case appdb.ISTATE_NOT_RESPONDING:
case appdb.ISTATE_EXITED:
checkAppHealth(app, done);
break;
default:
done();
break;
}
}, callback);
async.each(apps, checkAppHealth, callback);
});
}
+36 -48
View File
@@ -314,20 +314,19 @@ function unregisterSubdomain(app, callback) {
});
}
// callback is called with error when something fatal happenned (and not when some error state is reached)
function processInstallState(app, callback) {
// updates the app object and the database
function updateApp(app, values, callback) {
for (var value in values) {
app[value] = values[value];
}
debug(app.id + ' code:' + app.installationState);
appdb.update(app.id, values, callback);
// updates the app object and the database
function updateApp(app, values, callback) {
for (var value in values) {
app[value] = values[value];
}
debug(app.id + ' code:' + app.installationState);
appdb.update(app.id, values, callback);
}
// callback is called with error when something fatal happenned (and not when some error state is reached)
function processInstallState(app, callback) {
switch (app.installationState) {
case appdb.ISTATE_PENDING_INSTALL:
case appdb.ISTATE_NGINX_ERROR:
@@ -433,30 +432,8 @@ function processInstallState(app, callback) {
});
break;
case appdb.ISTATE_EXITED:
case appdb.ISTATE_CREATED_VOLUME:
case appdb.ISTATE_STARTING_CONTAINER:
case appdb.ISTATE_CONTAINER_ERROR:
appdb.getPortBindings(app.id, function (error, portBindings) {
if (error) return callback(error);
updateApp(app, { installationState: appdb.ISTATE_STARTING_CONTAINER }, function (error) {
if (error) return callback(error);
startContainer(app, portBindings, function (error) {
if (error) {
debug('Error creating container:' + error);
return updateApp(app, { installationState: appdb.ISTATE_CONTAINER_ERROR }, callback);
}
updateApp(app, { installationState: appdb.ISTATE_STARTED_CONTAINER }, callback);
});
});
});
break;
case appdb.ISTATE_STARTED_CONTAINER:
updateApp(app, { installationState: appdb.ISTATE_RUNNING }, callback);
updateApp(app, { installationState: appdb.ISTATE_INSTALLED }, callback);
break;
case appdb.ISTATE_PENDING_UNINSTALL:
@@ -468,11 +445,8 @@ function processInstallState(app, callback) {
});
break;
case appdb.ISTATE_RUNNING:
case appdb.ISTATE_NOT_RESPONDING:
case appdb.ISTATE_EXITED:
default:
assert(true, 'Should not reach this state: ' + app.installationState);
break;
}
}
@@ -481,8 +455,9 @@ function installApp(app, callback) {
processInstallState(app, function (error) {
if (error) return callback(error); // fatal error (not install error)
if (app.installationState === appdb.ISTATE_UNINSTALLED) {
debug('app uninstalled, stopping');
if (app.installationState === appdb.ISTATE_UNINSTALLED
|| app.installationState === appdb.ISTATE_INSTALLED) {
debug('app ' + app.installationState + ', stopping');
return callback(null);
}
@@ -491,20 +466,33 @@ function installApp(app, callback) {
return callback(null);
}
// move on
if (app.installationState === appdb.ISTATE_RUNNING || app.installationState === appdb.ISTATE_NOT_RESPONDING || app.installationState === appdb.ISTATE_EXITED) {
debug('app installed, stopping');
return callback(null);
}
installApp(app, callback);
});
}
function runApp(app, callback) {
appdb.getPortBindings(app.id, function (error, portBindings) {
if (error) return callback(error);
startContainer(app, portBindings, function (error) {
if (error) {
debug('Error creating container:' + error);
return updateApp(app, { runState: appdb.RSTATE_ERROR }, callback);
}
updateApp(app, { runState: appdb.RSTATE_RUNNING }, callback);
});
});
}
function run(appId, callback) {
appdb.get(appId, function (error, app) {
if (error) return callback(error);
installApp(app, callback);
installApp(app, function (error) {
if (error) return callback(error);
runApp(app, callback);
});
});
}
+1
View File
@@ -29,6 +29,7 @@ CREATE TABLE IF NOT EXISTS clients(
CREATE TABLE IF NOT EXISTS apps(
id VARCHAR(512) NOT NULL UNIQUE,
installationState VARCHAR(512) NOT NULL,
runState VARCHAR(512),
containerId VARCHAR(128),
manifestJson VARCHAR,
httpPort INTEGER,
+2
View File
@@ -236,6 +236,7 @@ describe('database', function () {
var APP_0 = {
id: 'appid-0',
installationState: 'some-status-0',
runState: null,
location: 'some-location-0',
manifestJson: null,
httpPort: null,
@@ -244,6 +245,7 @@ describe('database', function () {
var APP_1 = {
id: 'appid-1',
installationState: 'some-status-1',
runState: null,
location: 'some-location-1',
manifestJson: null,
httpPort: null,