From a37005d445a2287105f4e46894c9588f6f6560ec Mon Sep 17 00:00:00 2001 From: Girish Ramakrishnan Date: Tue, 8 Apr 2014 18:49:57 -0700 Subject: [PATCH] Revert the split of volume and syncer code syncer code is now considered core functionality. There were some problems with making the syncer code an app. syncer needs exclusive access to volumes. The volumes have a specific directory format (repo/) and the syncer cannot know if random apps write to the volume outside it's REST API. This means that if syncer were just an app, syncer's volumes cannot be shared with other apps. This would make simple cases like gallery app modifying pictures impossible. Additionally, docker was supposed to be used for mounting all the user's volumes into the app. However, docker does not have a way to add additional mounts on running containers (minor issue). It would also require to run an app instance per user. The new strategy is to make syncer a filesystem API. All read/write in the system goes through syncer's REST API. This means that we need only one app instance for multiple users. Volumes don't need to be mounted, since the app just uses REST calls. --- auth/routes/user.js | 2 - package.json | 2 +- {volume => sync}/config.js | 0 sync/routes/index.js | 3 +- sync/routes/sync.js | 4 +- {volume => sync}/routes/test/volume-test.js | 0 {volume => sync}/routes/volume.js | 0 sync/server.js | 38 +++- {volume => sync}/test/volume-test.js | 0 {volume => sync}/volume.js | 0 volume/routes/index.js | 7 - volume/server.js | 239 -------------------- 12 files changed, 32 insertions(+), 263 deletions(-) rename {volume => sync}/config.js (100%) rename {volume => sync}/routes/test/volume-test.js (100%) rename {volume => sync}/routes/volume.js (100%) rename {volume => sync}/test/volume-test.js (100%) rename {volume => sync}/volume.js (100%) delete mode 100644 volume/routes/index.js delete mode 100644 volume/server.js diff --git a/auth/routes/user.js b/auth/routes/user.js index 2fe723015..a7b2158fa 100644 --- a/auth/routes/user.js +++ b/auth/routes/user.js @@ -3,8 +3,6 @@ var db = require('../database'), DatabaseError = db.DatabaseError, user = require('../user'), - Volume = require('../../volume/volume.js'), - VolumeError = Volume.VolumeError, UserError = user.UserError, crypto = require('crypto'), async = require('async'), diff --git a/package.json b/package.json index dbbf43634..211da5c75 100644 --- a/package.json +++ b/package.json @@ -49,6 +49,6 @@ "expect.js": "*" }, "scripts": { - "test": "./node_modules/istanbul/lib/cli.js test $1 ./node_modules/mocha/bin/_mocha -- -R spec ./test ./sync/test ./sync/routes/test ./volume/test ./volume/routes/test ./auth/test ./auth/routes/test ./identity/test" + "test": "./node_modules/istanbul/lib/cli.js test $1 ./node_modules/mocha/bin/_mocha -- -R spec ./test ./sync/test ./sync/routes/test ./auth/test ./auth/routes/test ./identity/test" } } diff --git a/volume/config.js b/sync/config.js similarity index 100% rename from volume/config.js rename to sync/config.js diff --git a/sync/routes/index.js b/sync/routes/index.js index 9b848687d..8ca06f936 100644 --- a/sync/routes/index.js +++ b/sync/routes/index.js @@ -4,6 +4,7 @@ exports = module.exports = { user: require('../../auth/routes/user.js'), file: require('./file.js'), sync: require('./sync.js'), - fileops: require('./fileops.js') + fileops: require('./fileops.js'), + volume: require('./volume.js') }; diff --git a/sync/routes/sync.js b/sync/routes/sync.js index 7a2f5922d..cd794ff50 100644 --- a/sync/routes/sync.js +++ b/sync/routes/sync.js @@ -12,7 +12,7 @@ var debug = require('debug')('server:routes/sync'), exports = module.exports = { initialize: initialize, - attachVolume: attachVolume, + attachRepo: attachRepo, requireMountedVolume: requireMountedVolume, diff: diff, delta: delta @@ -26,7 +26,7 @@ function initialize(cfg) { config = cfg; } -function attachVolume(req, res, next, volumeId) { +function attachRepo(req, res, next, volumeId) { if (!volumeId) return next(new HttpError(400, 'Volume not specified')); var mountDir = path.join(config.mountRoot, volumeId); diff --git a/volume/routes/test/volume-test.js b/sync/routes/test/volume-test.js similarity index 100% rename from volume/routes/test/volume-test.js rename to sync/routes/test/volume-test.js diff --git a/volume/routes/volume.js b/sync/routes/volume.js similarity index 100% rename from volume/routes/volume.js rename to sync/routes/volume.js diff --git a/sync/server.js b/sync/server.js index c04a782af..007c1dafa 100644 --- a/sync/server.js +++ b/sync/server.js @@ -179,22 +179,37 @@ Server.prototype._initialize = function (callback) { that.app.get('/api/v1/user/info', routes.user.info); that.app.get('/api/v1/user/list', routes.user.list); - that.app.param('volume', routes.sync.attachVolume); + that.app.param('syncerVolume', routes.sync.attachRepo); - that.app.post('/api/v1/sync/:volume/diff', routes.sync.requireMountedVolume, routes.sync.diff); - that.app.post('/api/v1/sync/:volume/delta', routes.sync.requireMountedVolume, routes.sync.delta); + that.app.post('/api/v1/sync/:syncerVolume/diff', routes.sync.requireMountedVolume, routes.sync.diff); + that.app.post('/api/v1/sync/:syncerVolume/delta', routes.sync.requireMountedVolume, routes.sync.delta); - that.app.get('/api/v1/revisions/:volume/*', routes.sync.requireMountedVolume, routes.file.revisions); - that.app.get('/api/v1/file/:volume/*', routes.sync.requireMountedVolume, routes.file.read); - that.app.get('/api/v1/metadata/:volume/*', routes.sync.requireMountedVolume, routes.file.metadata); - that.app.put('/api/v1/file/:volume/*', routes.sync.requireMountedVolume, + that.app.get('/api/v1/revisions/:syncerVolume/*', routes.sync.requireMountedVolume, routes.file.revisions); + that.app.get('/api/v1/file/:syncerVolume/*', routes.sync.requireMountedVolume, routes.file.read); + that.app.get('/api/v1/metadata/:syncerVolume/*', routes.sync.requireMountedVolume, routes.file.metadata); + that.app.put('/api/v1/file/:syncerVolume/*', routes.sync.requireMountedVolume, routes.file.multipart({ maxFieldsSize: FIELD_LIMIT, limit: FILE_SIZE_LIMIT, timeout: FILE_TIMEOUT }), routes.file.putFile); - that.app.post('/api/v1/fileops/:volume/copy', routes.sync.requireMountedVolume, express.json({ strict: true }), routes.fileops.copy); - that.app.post('/api/v1/fileops/:volume/move', routes.sync.requireMountedVolume, express.json({ strict: true }), routes.fileops.move); - that.app.post('/api/v1/fileops/:volume/delete', routes.sync.requireMountedVolume, express.json({ strict: true }), routes.fileops.remove); - that.app.post('/api/v1/fileops/:volume/create_dir', routes.sync.requireMountedVolume, express.json({ strict: true }), routes.fileops.createDirectory); + that.app.post('/api/v1/fileops/:syncerVolume/copy', routes.sync.requireMountedVolume, express.json({ strict: true }), routes.fileops.copy); + that.app.post('/api/v1/fileops/:syncerVolume/move', routes.sync.requireMountedVolume, express.json({ strict: true }), routes.fileops.move); + that.app.post('/api/v1/fileops/:syncerVolume/delete', routes.sync.requireMountedVolume, express.json({ strict: true }), routes.fileops.remove); + that.app.post('/api/v1/fileops/:syncerVolume/create_dir', routes.sync.requireMountedVolume, express.json({ strict: true }), routes.fileops.createDirectory); + + // volume related routes + that.app.param('volume', routes.volume.attachVolume); + + that.app.get('/api/v1/volume/:volume/list', routes.volume.requireMountedVolume, routes.volume.listFiles); + that.app.get('/api/v1/volume/:volume/list/*', routes.volume.requireMountedVolume, routes.volume.listFiles); + that.app.get('/api/v1/volume/list', routes.volume.listVolumes); + that.app.post('/api/v1/volume/create', that._requirePassword.bind(that), routes.volume.createVolume); + that.app.post('/api/v1/volume/:volume/delete', that._requirePassword.bind(that), routes.volume.deleteVolume); + that.app.post('/api/v1/volume/:volume/mount', that._requirePassword.bind(that), routes.volume.mount); + that.app.post('/api/v1/volume/:volume/unmount', routes.volume.unmount); + that.app.get('/api/v1/volume/:volume/ismounted', routes.volume.isMounted); + that.app.get('/api/v1/volume/:volume/users', routes.volume.listUsers); + that.app.post('/api/v1/volume/:volume/users', routes.volume.addUser); + that.app.del('/api/v1/volume/:volume/users/:username', routes.volume.removeUser); }); this.app.set('port', that.config.port); @@ -215,6 +230,7 @@ Server.prototype._initialize = function (callback) { return callback(new Error('Error initializing database')); } + routes.volume.initialize(that.config); routes.sync.initialize(that.config); routes.user.initialize(that.config); diff --git a/volume/test/volume-test.js b/sync/test/volume-test.js similarity index 100% rename from volume/test/volume-test.js rename to sync/test/volume-test.js diff --git a/volume/volume.js b/sync/volume.js similarity index 100% rename from volume/volume.js rename to sync/volume.js diff --git a/volume/routes/index.js b/volume/routes/index.js deleted file mode 100644 index 4431ae9a5..000000000 --- a/volume/routes/index.js +++ /dev/null @@ -1,7 +0,0 @@ -'use strict'; - -exports = module.exports = { - user: require('../../auth/routes/user.js'), - volume: require('./volume.js') -}; - diff --git a/volume/server.js b/volume/server.js deleted file mode 100644 index d38461e6b..000000000 --- a/volume/server.js +++ /dev/null @@ -1,239 +0,0 @@ -'use strict'; - -var express = require('express'), - http = require('http'), - HttpError = require('../common/httperror'), - HttpSuccess = require('../common/httpsuccess'), - path = require('path'), - fs = require('fs'), - mkdirp = require('mkdirp'), - db = require('../auth/database.js'), - routes = require('./routes/index.js'), - debug = require('debug')('server:server'), - assert = require('assert'), - util = require('util'), - pkg = require('./../package.json'); - -exports = module.exports = Server; - -function Server(config) { - assert(typeof config === 'object'); - - this.config = config; - this.app = null; -} - - -// Success handler -Server.prototype._successHandler = function (success, req, res, next) { - if (success instanceof HttpSuccess) { - debug('Send response with status', success.statusCode, 'and body', success.body); - res.send(success.statusCode, success.body); - } else { - next(success); - } -}; - - -// Error handlers. These are called until one of them sends headers -Server.prototype._clientErrorHandler = function (err, req, res, next) { - var status = err.status || err.statusCode; // connect/express or our app - - // if the request took too long, assume it's a problem on the client - if (err.timeout && err.status == 503) { // timeout() middleware - status = 408; - } - - if (status >= 400 && status <= 499) { - res.send(status, { status: http.STATUS_CODES[status], message: err.message }); - debug(http.STATUS_CODES[status] + ' : ' + err.message); - debug(err.stack); - } else { - next(err); - } -}; - -Server.prototype._serverErrorHandler = function (err, req, res, next) { - var status = err.status || err.statusCode || 500; - res.send(status, { status: http.STATUS_CODES[status], message: err.message }); - console.error(http.STATUS_CODES[status] + ' : ' + err.message); - console.error(err.stack); -}; - -/* - Middleware which makes the route require a password in the body besides a token. -*/ -Server.prototype._requirePassword = function (req, res, next) { - if (!req.body.password) return next(new HttpError(400, 'API call requires the users password.')); - next(); -}; - - -/* - Middleware which makes the route only accessable for the admin user. -*/ -Server.prototype._requireAdmin = function (req, res, next) { - if (!req.user.admin) return next(new HttpError(403, 'API call requires the admin rights.')); - next(); -}; - -Server.prototype._loadMiddleware = function () { - var middleware = { }; - // TODO that folder lookup is a bit silly maybe with the '../' - Johannes - fs.readdirSync(__dirname + '/../middleware').forEach(function (filename) { - if (!/\.js$/.test(filename)) return; - var name = path.basename(filename, '.js'); - function load() { return require('./../middleware/' + name); } - middleware.__defineGetter__(name, load); - }); - return middleware; -}; - -Server.prototype._initialize = function (callback) { - var that = this; - var middleware = this._loadMiddleware(); - this.app = express(); - - this.app.configure(function () { - var QUERY_LIMIT = '10mb', // max size for json and urlencoded queries - FIELD_LIMIT = 2 * 1024, // max fields that can appear in multipart - FILE_SIZE_LIMIT = '521mb', // max file size that can be uploaded - UPLOAD_LIMIT = '521mb'; // catch all max size for any type of request - - var REQUEST_TIMEOUT = 10000, // timeout for all requests - FILE_TIMEOUT = 3 * 60 * 1000; // increased timeout for file uploads (3 mins) - - var json = express.json({ strict: true, limit: QUERY_LIMIT }), // application/json - urlencoded = express.urlencoded({ limit: QUERY_LIMIT }); // application/x-www-form-urlencoded - - if (!that.config.silent) { - that.app.use(express.logger({ format: 'dev', immediate: false })); - } - - that.app - .use(express.timeout(REQUEST_TIMEOUT)) - .use(express.limit(UPLOAD_LIMIT)) - .use(json) - .use(urlencoded) - .use(express.cookieParser()) - // API calls that do not require authorization - .use(middleware.cors({ origins: [ '*' ], allowCredentials: true })) - .use(middleware.contentType('application/json')) - .use('/api/v1/createadmin', routes.user.createAdmin) // ## FIXME: allow this before auth for now - .use(routes.user.authenticate) - .use(that.app.router) - .use(that._successHandler.bind(that)) - .use(that._clientErrorHandler.bind(that)) - .use(that._serverErrorHandler.bind(that)); - - // routes controlled by app.router - that.app.post('/api/v1/token', routes.user.createToken); // TODO remove that route - that.app.get('/api/v1/user/token', routes.user.createToken); - that.app.get('/api/v1/logout', routes.user.logout); // TODO remove that route - that.app.get('/api/v1/user/logout', routes.user.logout); - that.app.post('/api/v1/user/create', that._requireAdmin.bind(that), routes.user.create); - that.app.post('/api/v1/user/remove', that._requireAdmin.bind(that), routes.user.remove); - that.app.post('/api/v1/user/password', that._requirePassword.bind(that), routes.user.changePassword); - that.app.get('/api/v1/user/info', routes.user.info); - that.app.get('/api/v1/user/list', routes.user.list); - - that.app.param('volume', routes.volume.attachVolume); - - that.app.get('/api/v1/volume/:volume/list', routes.volume.requireMountedVolume, routes.volume.listFiles); - that.app.get('/api/v1/volume/:volume/list/*', routes.volume.requireMountedVolume, routes.volume.listFiles); - that.app.get('/api/v1/volume/list', routes.volume.listVolumes); - that.app.post('/api/v1/volume/create', that._requirePassword.bind(that), routes.volume.createVolume); - that.app.post('/api/v1/volume/:volume/delete', that._requirePassword.bind(that), routes.volume.deleteVolume); - that.app.post('/api/v1/volume/:volume/mount', that._requirePassword.bind(that), routes.volume.mount); - that.app.post('/api/v1/volume/:volume/unmount', routes.volume.unmount); - that.app.get('/api/v1/volume/:volume/ismounted', routes.volume.isMounted); - that.app.get('/api/v1/volume/:volume/users', routes.volume.listUsers); - that.app.post('/api/v1/volume/:volume/users', routes.volume.addUser); - that.app.del('/api/v1/volume/:volume/users/:username', routes.volume.removeUser); - }); - - this.app.set('port', that.config.port); - - if (!that.config.silent) { - console.log('Server listening on port ' + this.app.get('port')); - console.log('Using data root:', that.config.dataRoot); - console.log('Using config root:', that.config.configRoot); - console.log('Using mount root:', that.config.mountRoot); - } - - // ensure data/config/mount paths - mkdirp.sync(that.config.dataRoot); - mkdirp.sync(that.config.configRoot); - mkdirp.sync(that.config.mountRoot); - - if (!db.initialize(that.config)) { - return callback(new Error('Error initializing database')); - } - - routes.volume.initialize(that.config); - routes.user.initialize(that.config); - - callback(null); -}; - -// TODO maybe we can get rid of that function and inline it - Johannes -Server.prototype._listen = function (callback) { - this.app.httpServer = http.createServer(this.app); - - function callbackWrapper(error) { - if (callback) { - callback(error); - callback = null; - } else { - console.error('Try to call back twice', error); - } - } - - this.app.httpServer.listen(this.app.get('port'), function (err) { - if (err) return callbackWrapper(err); - callbackWrapper(); - }); - - this.app.httpServer.on('error', function (err) { - callbackWrapper(err); - }); -}; - -Server.prototype.start = function (callback) { - assert(typeof callback === 'function'); - - var that = this; - - if (this.app) { - return callback(new Error('Server is already up and running.')); - } - - this._initialize(function (err) { - if (err) return callback(err); - - that._listen(function (err) { - if (err) return callback(err); - - callback(null); - }); - }); -}; - -Server.prototype.stop = function (callback) { - // Any other way to check if app is an object we expect? - assert(typeof callback === 'function'); - - var that = this; - - if (!this.app.httpServer) { - return callback(); - } - - this.app.httpServer.close(function () { - that.app.httpServer.unref(); - // TODO should delete the app variable - that.app = null; - - callback(); - }); -};