jjkiers/cloudron-box
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Revert the split of volume and syncer code

Browse Source 
syncer code is now considered core functionality. There were some
problems with making the syncer code an app.

syncer needs exclusive access to volumes. The volumes have a specific
directory format (repo/) and the syncer cannot know if random
apps write to the volume outside it's REST API. This means that
if syncer were just an app, syncer's volumes cannot be shared
with other apps. This would make simple cases like gallery app
modifying pictures impossible. Additionally, docker was supposed
to be used for mounting all the user's volumes into the app.
However, docker does not have a way to add additional mounts on
running containers (minor issue). It would also require to run
an app instance per user.

The new strategy is to make syncer a filesystem API. All read/write
in the system goes through syncer's REST API. This means that we
need only one app instance for multiple users. Volumes don't need
to be mounted, since the app just uses REST calls.
This commit is contained in:
Girish Ramakrishnan
2014-04-08 18:49:57 -07:00
parent a9af314a62
commit a37005d445
12 changed files with 32 additions and 263 deletions
Download Patch File Download Diff File Expand all files Collapse all files
auth/routes/user.js
-2
View File
@@ -3,8 +3,6 @@
var db = require('../database'),
DatabaseError = db.DatabaseError,
user = require('../user'),
Volume = require('../../volume/volume.js'),
VolumeError = Volume.VolumeError,
UserError = user.UserError,
crypto = require('crypto'),
async = require('async'),
package.json
+1 -1
View File
@@ -49,6 +49,6 @@
"expect.js": "*"
},
"scripts": {
"test": "./node_modules/istanbul/lib/cli.js test $1 ./node_modules/mocha/bin/_mocha -- -R spec ./test ./sync/test ./sync/routes/test ./volume/test ./volume/routes/test ./auth/test ./auth/routes/test ./identity/test"
"test": "./node_modules/istanbul/lib/cli.js test $1 ./node_modules/mocha/bin/_mocha -- -R spec ./test ./sync/test ./sync/routes/test ./auth/test ./auth/routes/test ./identity/test"
}
}
volume/config.js → sync/config.js
View File
sync/routes/index.js
+2 -1
View File
@@ -4,6 +4,7 @@ exports = module.exports = {
user: require('../../auth/routes/user.js'),
file: require('./file.js'),
sync: require('./sync.js'),
fileops: require('./fileops.js')
fileops: require('./fileops.js'),
volume: require('./volume.js')
};
sync/routes/sync.js
+2 -2
View File
@@ -12,7 +12,7 @@ var debug = require('debug')('server:routes/sync'),
exports = module.exports = {
initialize: initialize,
attachVolume: attachVolume,
attachRepo: attachRepo,
requireMountedVolume: requireMountedVolume,
diff: diff,
delta: delta
@@ -26,7 +26,7 @@ function initialize(cfg) {
config = cfg;
}
function attachVolume(req, res, next, volumeId) {
function attachRepo(req, res, next, volumeId) {
if (!volumeId) return next(new HttpError(400, 'Volume not specified'));
var mountDir = path.join(config.mountRoot, volumeId);
volume/routes/test/volume-test.js → sync/routes/test/volume-test.js
View File
volume/routes/volume.js → sync/routes/volume.js
View File
sync/server.js
+27 -11
View File
@@ -179,22 +179,37 @@ Server.prototype._initialize = function (callback) {
that.app.get('/api/v1/user/info', routes.user.info);
that.app.get('/api/v1/user/list', routes.user.list);
that.app.param('volume', routes.sync.attachVolume);
that.app.param('syncerVolume', routes.sync.attachRepo);
that.app.post('/api/v1/sync/:volume/diff', routes.sync.requireMountedVolume, routes.sync.diff);
that.app.post('/api/v1/sync/:volume/delta', routes.sync.requireMountedVolume, routes.sync.delta);
that.app.post('/api/v1/sync/:syncerVolume/diff', routes.sync.requireMountedVolume, routes.sync.diff);
that.app.post('/api/v1/sync/:syncerVolume/delta', routes.sync.requireMountedVolume, routes.sync.delta);
that.app.get('/api/v1/revisions/:volume/*', routes.sync.requireMountedVolume, routes.file.revisions);
that.app.get('/api/v1/file/:volume/*', routes.sync.requireMountedVolume, routes.file.read);
that.app.get('/api/v1/metadata/:volume/*', routes.sync.requireMountedVolume, routes.file.metadata);
that.app.put('/api/v1/file/:volume/*', routes.sync.requireMountedVolume,
that.app.get('/api/v1/revisions/:syncerVolume/*', routes.sync.requireMountedVolume, routes.file.revisions);
that.app.get('/api/v1/file/:syncerVolume/*', routes.sync.requireMountedVolume, routes.file.read);
that.app.get('/api/v1/metadata/:syncerVolume/*', routes.sync.requireMountedVolume, routes.file.metadata);
that.app.put('/api/v1/file/:syncerVolume/*', routes.sync.requireMountedVolume,
routes.file.multipart({ maxFieldsSize: FIELD_LIMIT, limit: FILE_SIZE_LIMIT, timeout: FILE_TIMEOUT }),
routes.file.putFile);
that.app.post('/api/v1/fileops/:volume/copy', routes.sync.requireMountedVolume, express.json({ strict: true }), routes.fileops.copy);
that.app.post('/api/v1/fileops/:volume/move', routes.sync.requireMountedVolume, express.json({ strict: true }), routes.fileops.move);
that.app.post('/api/v1/fileops/:volume/delete', routes.sync.requireMountedVolume, express.json({ strict: true }), routes.fileops.remove);
that.app.post('/api/v1/fileops/:volume/create_dir', routes.sync.requireMountedVolume, express.json({ strict: true }), routes.fileops.createDirectory);
that.app.post('/api/v1/fileops/:syncerVolume/copy', routes.sync.requireMountedVolume, express.json({ strict: true }), routes.fileops.copy);
that.app.post('/api/v1/fileops/:syncerVolume/move', routes.sync.requireMountedVolume, express.json({ strict: true }), routes.fileops.move);
that.app.post('/api/v1/fileops/:syncerVolume/delete', routes.sync.requireMountedVolume, express.json({ strict: true }), routes.fileops.remove);
that.app.post('/api/v1/fileops/:syncerVolume/create_dir', routes.sync.requireMountedVolume, express.json({ strict: true }), routes.fileops.createDirectory);
// volume related routes
that.app.param('volume', routes.volume.attachVolume);
that.app.get('/api/v1/volume/:volume/list', routes.volume.requireMountedVolume, routes.volume.listFiles);
that.app.get('/api/v1/volume/:volume/list/*', routes.volume.requireMountedVolume, routes.volume.listFiles);
that.app.get('/api/v1/volume/list', routes.volume.listVolumes);
that.app.post('/api/v1/volume/create', that._requirePassword.bind(that), routes.volume.createVolume);
that.app.post('/api/v1/volume/:volume/delete', that._requirePassword.bind(that), routes.volume.deleteVolume);
that.app.post('/api/v1/volume/:volume/mount', that._requirePassword.bind(that), routes.volume.mount);
that.app.post('/api/v1/volume/:volume/unmount', routes.volume.unmount);
that.app.get('/api/v1/volume/:volume/ismounted', routes.volume.isMounted);
that.app.get('/api/v1/volume/:volume/users', routes.volume.listUsers);
that.app.post('/api/v1/volume/:volume/users', routes.volume.addUser);
that.app.del('/api/v1/volume/:volume/users/:username', routes.volume.removeUser);
});
this.app.set('port', that.config.port);
@@ -215,6 +230,7 @@ Server.prototype._initialize = function (callback) {
return callback(new Error('Error initializing database'));
}
routes.volume.initialize(that.config);
routes.sync.initialize(that.config);
routes.user.initialize(that.config);
volume/test/volume-test.js → sync/test/volume-test.js
View File
volume/volume.js → sync/volume.js
View File
volume/routes/index.js
-7
View File
@@ -1,7 +0,0 @@
'use strict';
exports = module.exports = {
user: require('../../auth/routes/user.js'),
volume: require('./volume.js')
};
volume/server.js
-239
View File
@@ -1,239 +0,0 @@
'use strict';
var express = require('express'),
http = require('http'),
HttpError = require('../common/httperror'),
HttpSuccess = require('../common/httpsuccess'),
path = require('path'),
fs = require('fs'),
mkdirp = require('mkdirp'),
db = require('../auth/database.js'),
routes = require('./routes/index.js'),
debug = require('debug')('server:server'),
assert = require('assert'),
util = require('util'),
pkg = require('./../package.json');
exports = module.exports = Server;
function Server(config) {
assert(typeof config === 'object');
this.config = config;
this.app = null;
}
// Success handler
Server.prototype._successHandler = function (success, req, res, next) {
if (success instanceof HttpSuccess) {
debug('Send response with status', success.statusCode, 'and body', success.body);
res.send(success.statusCode, success.body);
} else {
next(success);
}
};
// Error handlers. These are called until one of them sends headers
Server.prototype._clientErrorHandler = function (err, req, res, next) {
var status = err.status || err.statusCode; // connect/express or our app
// if the request took too long, assume it's a problem on the client
if (err.timeout && err.status == 503) { // timeout() middleware
status = 408;
}
if (status >= 400 && status <= 499) {
res.send(status, { status: http.STATUS_CODES[status], message: err.message });
debug(http.STATUS_CODES[status] + ' : ' + err.message);
debug(err.stack);
} else {
next(err);
}
};
Server.prototype._serverErrorHandler = function (err, req, res, next) {
var status = err.status || err.statusCode || 500;
res.send(status, { status: http.STATUS_CODES[status], message: err.message });
console.error(http.STATUS_CODES[status] + ' : ' + err.message);
console.error(err.stack);
};
/*
Middleware which makes the route require a password in the body besides a token.
*/
Server.prototype._requirePassword = function (req, res, next) {
if (!req.body.password) return next(new HttpError(400, 'API call requires the users password.'));
next();
};
/*
Middleware which makes the route only accessable for the admin user.
*/
Server.prototype._requireAdmin = function (req, res, next) {
if (!req.user.admin) return next(new HttpError(403, 'API call requires the admin rights.'));
next();
};
Server.prototype._loadMiddleware = function () {
var middleware = { };
// TODO that folder lookup is a bit silly maybe with the '../' - Johannes
fs.readdirSync(__dirname + '/../middleware').forEach(function (filename) {
if (!/\.js$/.test(filename)) return;
var name = path.basename(filename, '.js');
function load() { return require('./../middleware/' + name); }
middleware.__defineGetter__(name, load);
});
return middleware;
};
Server.prototype._initialize = function (callback) {
var that = this;
var middleware = this._loadMiddleware();
this.app = express();
this.app.configure(function () {
var QUERY_LIMIT = '10mb', // max size for json and urlencoded queries
FIELD_LIMIT = 2 * 1024, // max fields that can appear in multipart
FILE_SIZE_LIMIT = '521mb', // max file size that can be uploaded
UPLOAD_LIMIT = '521mb'; // catch all max size for any type of request
var REQUEST_TIMEOUT = 10000, // timeout for all requests
FILE_TIMEOUT = 3 * 60 * 1000; // increased timeout for file uploads (3 mins)
var json = express.json({ strict: true, limit: QUERY_LIMIT }), // application/json
urlencoded = express.urlencoded({ limit: QUERY_LIMIT }); // application/x-www-form-urlencoded
if (!that.config.silent) {
that.app.use(express.logger({ format: 'dev', immediate: false }));
}
that.app
.use(express.timeout(REQUEST_TIMEOUT))
.use(express.limit(UPLOAD_LIMIT))
.use(json)
.use(urlencoded)
.use(express.cookieParser())
// API calls that do not require authorization
.use(middleware.cors({ origins: [ '*' ], allowCredentials: true }))
.use(middleware.contentType('application/json'))
.use('/api/v1/createadmin', routes.user.createAdmin) // ## FIXME: allow this before auth for now
.use(routes.user.authenticate)
.use(that.app.router)
.use(that._successHandler.bind(that))
.use(that._clientErrorHandler.bind(that))
.use(that._serverErrorHandler.bind(that));
// routes controlled by app.router
that.app.post('/api/v1/token', routes.user.createToken); // TODO remove that route
that.app.get('/api/v1/user/token', routes.user.createToken);
that.app.get('/api/v1/logout', routes.user.logout); // TODO remove that route
that.app.get('/api/v1/user/logout', routes.user.logout);
that.app.post('/api/v1/user/create', that._requireAdmin.bind(that), routes.user.create);
that.app.post('/api/v1/user/remove', that._requireAdmin.bind(that), routes.user.remove);
that.app.post('/api/v1/user/password', that._requirePassword.bind(that), routes.user.changePassword);
that.app.get('/api/v1/user/info', routes.user.info);
that.app.get('/api/v1/user/list', routes.user.list);
that.app.param('volume', routes.volume.attachVolume);
that.app.get('/api/v1/volume/:volume/list', routes.volume.requireMountedVolume, routes.volume.listFiles);
that.app.get('/api/v1/volume/:volume/list/*', routes.volume.requireMountedVolume, routes.volume.listFiles);
that.app.get('/api/v1/volume/list', routes.volume.listVolumes);
that.app.post('/api/v1/volume/create', that._requirePassword.bind(that), routes.volume.createVolume);
that.app.post('/api/v1/volume/:volume/delete', that._requirePassword.bind(that), routes.volume.deleteVolume);
that.app.post('/api/v1/volume/:volume/mount', that._requirePassword.bind(that), routes.volume.mount);
that.app.post('/api/v1/volume/:volume/unmount', routes.volume.unmount);
that.app.get('/api/v1/volume/:volume/ismounted', routes.volume.isMounted);
that.app.get('/api/v1/volume/:volume/users', routes.volume.listUsers);
that.app.post('/api/v1/volume/:volume/users', routes.volume.addUser);
that.app.del('/api/v1/volume/:volume/users/:username', routes.volume.removeUser);
});
this.app.set('port', that.config.port);
if (!that.config.silent) {
console.log('Server listening on port ' + this.app.get('port'));
console.log('Using data root:', that.config.dataRoot);
console.log('Using config root:', that.config.configRoot);
console.log('Using mount root:', that.config.mountRoot);
}
// ensure data/config/mount paths
mkdirp.sync(that.config.dataRoot);
mkdirp.sync(that.config.configRoot);
mkdirp.sync(that.config.mountRoot);
if (!db.initialize(that.config)) {
return callback(new Error('Error initializing database'));
}
routes.volume.initialize(that.config);
routes.user.initialize(that.config);
callback(null);
};
// TODO maybe we can get rid of that function and inline it - Johannes
Server.prototype._listen = function (callback) {
this.app.httpServer = http.createServer(this.app);
function callbackWrapper(error) {
if (callback) {
callback(error);
callback = null;
} else {
console.error('Try to call back twice', error);
}
}
this.app.httpServer.listen(this.app.get('port'), function (err) {
if (err) return callbackWrapper(err);
callbackWrapper();
});
this.app.httpServer.on('error', function (err) {
callbackWrapper(err);
});
};
Server.prototype.start = function (callback) {
assert(typeof callback === 'function');
var that = this;
if (this.app) {
return callback(new Error('Server is already up and running.'));
}
this._initialize(function (err) {
if (err) return callback(err);
that._listen(function (err) {
if (err) return callback(err);
callback(null);
});
});
};
Server.prototype.stop = function (callback) {
// Any other way to check if app is an object we expect?
assert(typeof callback === 'function');
var that = this;
if (!this.app.httpServer) {
return callback();
}
this.app.httpServer.close(function () {
that.app.httpServer.unref();
// TODO should delete the app variable
that.app = null;
callback();
});
};