Implement runtime state
This separation makes it possible for apphealthtask to go in parallel and not worry about it the app being uninstalled. Fixes #27
This commit is contained in:
+7
-7
@@ -31,13 +31,10 @@ exports = module.exports = {
|
||||
ISTATE_DOWNLOADING_IMAGE: 'downloading_image',
|
||||
ISTATE_DOWNLOADED_IMAGE: 'downloaded_image',
|
||||
ISTATE_IMAGE_ERROR: 'image_error',
|
||||
|
||||
ISTATE_CREATING_CONTAINER: 'creating_container',
|
||||
ISTATE_CREATED_CONTAINER: 'created_container',
|
||||
|
||||
ISTATE_STARTING_CONTAINER: 'starting_container',
|
||||
ISTATE_STARTED_CONTAINER: 'started_container',
|
||||
ISTATE_CONTAINER_ERROR: 'container_error',
|
||||
|
||||
ISTATE_CREATING_VOLUME: 'creating_volume',
|
||||
ISTATE_CREATED_VOLUME: 'created_volume',
|
||||
ISTATE_VOLUME_ERROR: 'volume_error',
|
||||
@@ -46,9 +43,12 @@ exports = module.exports = {
|
||||
ISTATE_REGISTERED_SUBDOMAIN: 'registered_subdomain',
|
||||
ISTATE_SUBDOMAIN_ERROR: 'subdomain_error',
|
||||
|
||||
ISTATE_RUNNING: 'running',
|
||||
ISTATE_EXITED: 'exited',
|
||||
ISTATE_NOT_RESPONDING: 'not_responding'
|
||||
ISTATE_INSTALLED: 'installed',
|
||||
|
||||
RSTATE_RUNNING: 'running',
|
||||
RSTATE_ERROR: 'error',
|
||||
RSTATE_EXITED: 'exited',
|
||||
RSTATE_NOT_RESPONDING: 'not_responding'
|
||||
};
|
||||
|
||||
var DatabaseError = require('./databaseerror'),
|
||||
|
||||
+22
-27
@@ -5,6 +5,7 @@
|
||||
var assert = require('assert'),
|
||||
config = require('../config.js'),
|
||||
database = require('./database.js'),
|
||||
DatabaseError = require('./databaseerror'),
|
||||
appdb = require('./appdb.js'),
|
||||
async = require('async'),
|
||||
Docker = require('dockerode'),
|
||||
@@ -38,29 +39,36 @@ function initialize() {
|
||||
});
|
||||
}
|
||||
|
||||
function updateApp(app, values, callback) {
|
||||
for (var value in values) {
|
||||
app[value] = values[value];
|
||||
}
|
||||
appdb.update(app.id, values, callback);
|
||||
}
|
||||
|
||||
// # TODO should probably poll from the outside network instead of the docker network?
|
||||
// callback is called with error for fatal errors and not if health check failed
|
||||
function checkAppHealth(app, callback) {
|
||||
if (app.installationState !== appdb.ISTATE_INSTALLED) return callback(null);
|
||||
|
||||
var container = docker.getContainer(app.containerId),
|
||||
manifest = JSON.parse(app.manifestJson); // this is guaranteed not to throw since it's already been verified in downloadManifest()
|
||||
|
||||
function updateApp(app, values, callback) {
|
||||
for (var value in values) {
|
||||
app[value] = values[value];
|
||||
}
|
||||
appdb.update(app.id, values, function (error) {
|
||||
if (error && error.reason === DatabaseError.NOT_FOUND) { // app got uninstalled
|
||||
return callback(null);
|
||||
}
|
||||
|
||||
callback(error);
|
||||
});
|
||||
}
|
||||
|
||||
container.inspect(function (err, data) {
|
||||
if (err || !data || !data.State) {
|
||||
debug('Error inspecting container');
|
||||
updateApp(app, { installationState: appdb.ISTATE_IMAGE_ERROR }, FATAL_CALLBACK);
|
||||
return callback(err);
|
||||
return updateApp(app, { runState: appdb.RSTATE_ERROR }, callback);
|
||||
}
|
||||
|
||||
if (data.State.Running !== true) {
|
||||
debug(app.id + ' has exited');
|
||||
updateApp(app, { installationState: appdb.ISTATE_EXITED }, FATAL_CALLBACK);
|
||||
return callback(null);
|
||||
return updateApp(app, { runState: appdb.RSTATE_EXITED }, callback);
|
||||
}
|
||||
|
||||
var healthCheckUrl = 'http://127.0.0.1:' + app.httpPort + manifest.health_check_url;
|
||||
@@ -71,12 +79,10 @@ function checkAppHealth(app, callback) {
|
||||
|
||||
if (error || res.status !== 200) {
|
||||
debug('Marking application as dead: ' + app.id);
|
||||
updateApp(app, { installationState: appdb.ISTATE_NOT_RESPONDING }, FATAL_CALLBACK);
|
||||
callback(null);
|
||||
updateApp(app, { runState: appdb.RSTATE_NOT_RESPONDING }, callback);
|
||||
} else {
|
||||
debug('healthy app:' + app.id);
|
||||
updateApp(app, { installationState: appdb.ISTATE_RUNNING }, FATAL_CALLBACK);
|
||||
callback(null);
|
||||
updateApp(app, { runState: appdb.RSTATE_RUNNING }, callback);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -86,18 +92,7 @@ function processApps(callback) {
|
||||
appdb.getAll(function (error, apps) {
|
||||
if (error) return callback(error);
|
||||
|
||||
async.each(apps, function (app, done) {
|
||||
switch (app.installationState) {
|
||||
case appdb.ISTATE_RUNNING:
|
||||
case appdb.ISTATE_NOT_RESPONDING:
|
||||
case appdb.ISTATE_EXITED:
|
||||
checkAppHealth(app, done);
|
||||
break;
|
||||
default:
|
||||
done();
|
||||
break;
|
||||
}
|
||||
}, callback);
|
||||
async.each(apps, checkAppHealth, callback);
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
+36
-48
@@ -314,20 +314,19 @@ function unregisterSubdomain(app, callback) {
|
||||
});
|
||||
}
|
||||
|
||||
// callback is called with error when something fatal happenned (and not when some error state is reached)
|
||||
function processInstallState(app, callback) {
|
||||
|
||||
// updates the app object and the database
|
||||
function updateApp(app, values, callback) {
|
||||
for (var value in values) {
|
||||
app[value] = values[value];
|
||||
}
|
||||
|
||||
debug(app.id + ' code:' + app.installationState);
|
||||
|
||||
appdb.update(app.id, values, callback);
|
||||
// updates the app object and the database
|
||||
function updateApp(app, values, callback) {
|
||||
for (var value in values) {
|
||||
app[value] = values[value];
|
||||
}
|
||||
|
||||
debug(app.id + ' code:' + app.installationState);
|
||||
|
||||
appdb.update(app.id, values, callback);
|
||||
}
|
||||
|
||||
// callback is called with error when something fatal happenned (and not when some error state is reached)
|
||||
function processInstallState(app, callback) {
|
||||
switch (app.installationState) {
|
||||
case appdb.ISTATE_PENDING_INSTALL:
|
||||
case appdb.ISTATE_NGINX_ERROR:
|
||||
@@ -433,30 +432,8 @@ function processInstallState(app, callback) {
|
||||
});
|
||||
break;
|
||||
|
||||
case appdb.ISTATE_EXITED:
|
||||
case appdb.ISTATE_CREATED_VOLUME:
|
||||
case appdb.ISTATE_STARTING_CONTAINER:
|
||||
case appdb.ISTATE_CONTAINER_ERROR:
|
||||
appdb.getPortBindings(app.id, function (error, portBindings) {
|
||||
if (error) return callback(error);
|
||||
|
||||
updateApp(app, { installationState: appdb.ISTATE_STARTING_CONTAINER }, function (error) {
|
||||
if (error) return callback(error);
|
||||
|
||||
startContainer(app, portBindings, function (error) {
|
||||
if (error) {
|
||||
debug('Error creating container:' + error);
|
||||
return updateApp(app, { installationState: appdb.ISTATE_CONTAINER_ERROR }, callback);
|
||||
}
|
||||
|
||||
updateApp(app, { installationState: appdb.ISTATE_STARTED_CONTAINER }, callback);
|
||||
});
|
||||
});
|
||||
});
|
||||
break;
|
||||
|
||||
case appdb.ISTATE_STARTED_CONTAINER:
|
||||
updateApp(app, { installationState: appdb.ISTATE_RUNNING }, callback);
|
||||
updateApp(app, { installationState: appdb.ISTATE_INSTALLED }, callback);
|
||||
break;
|
||||
|
||||
case appdb.ISTATE_PENDING_UNINSTALL:
|
||||
@@ -468,11 +445,8 @@ function processInstallState(app, callback) {
|
||||
});
|
||||
break;
|
||||
|
||||
case appdb.ISTATE_RUNNING:
|
||||
case appdb.ISTATE_NOT_RESPONDING:
|
||||
case appdb.ISTATE_EXITED:
|
||||
default:
|
||||
assert(true, 'Should not reach this state: ' + app.installationState);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -481,8 +455,9 @@ function installApp(app, callback) {
|
||||
processInstallState(app, function (error) {
|
||||
if (error) return callback(error); // fatal error (not install error)
|
||||
|
||||
if (app.installationState === appdb.ISTATE_UNINSTALLED) {
|
||||
debug('app uninstalled, stopping');
|
||||
if (app.installationState === appdb.ISTATE_UNINSTALLED
|
||||
|| app.installationState === appdb.ISTATE_INSTALLED) {
|
||||
debug('app ' + app.installationState + ', stopping');
|
||||
return callback(null);
|
||||
}
|
||||
|
||||
@@ -491,20 +466,33 @@ function installApp(app, callback) {
|
||||
return callback(null);
|
||||
}
|
||||
|
||||
// move on
|
||||
if (app.installationState === appdb.ISTATE_RUNNING || app.installationState === appdb.ISTATE_NOT_RESPONDING || app.installationState === appdb.ISTATE_EXITED) {
|
||||
debug('app installed, stopping');
|
||||
return callback(null);
|
||||
}
|
||||
|
||||
installApp(app, callback);
|
||||
});
|
||||
}
|
||||
|
||||
function runApp(app, callback) {
|
||||
appdb.getPortBindings(app.id, function (error, portBindings) {
|
||||
if (error) return callback(error);
|
||||
|
||||
startContainer(app, portBindings, function (error) {
|
||||
if (error) {
|
||||
debug('Error creating container:' + error);
|
||||
return updateApp(app, { runState: appdb.RSTATE_ERROR }, callback);
|
||||
}
|
||||
|
||||
updateApp(app, { runState: appdb.RSTATE_RUNNING }, callback);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function run(appId, callback) {
|
||||
appdb.get(appId, function (error, app) {
|
||||
if (error) return callback(error);
|
||||
installApp(app, callback);
|
||||
installApp(app, function (error) {
|
||||
if (error) return callback(error);
|
||||
|
||||
runApp(app, callback);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -29,6 +29,7 @@ CREATE TABLE IF NOT EXISTS clients(
|
||||
CREATE TABLE IF NOT EXISTS apps(
|
||||
id VARCHAR(512) NOT NULL UNIQUE,
|
||||
installationState VARCHAR(512) NOT NULL,
|
||||
runState VARCHAR(512),
|
||||
containerId VARCHAR(128),
|
||||
manifestJson VARCHAR,
|
||||
httpPort INTEGER,
|
||||
|
||||
@@ -236,6 +236,7 @@ describe('database', function () {
|
||||
var APP_0 = {
|
||||
id: 'appid-0',
|
||||
installationState: 'some-status-0',
|
||||
runState: null,
|
||||
location: 'some-location-0',
|
||||
manifestJson: null,
|
||||
httpPort: null,
|
||||
@@ -244,6 +245,7 @@ describe('database', function () {
|
||||
var APP_1 = {
|
||||
id: 'appid-1',
|
||||
installationState: 'some-status-1',
|
||||
runState: null,
|
||||
location: 'some-location-1',
|
||||
manifestJson: null,
|
||||
httpPort: null,
|
||||
|
||||
Reference in New Issue
Block a user