diff --git a/docs/references/api.md b/docs/references/api.md index 84c29e2e4..58b14d581 100644 --- a/docs/references/api.md +++ b/docs/references/api.md @@ -119,7 +119,8 @@ Request: memoryLimit: , // memory constraint in bytes backupId: , // initialize the app from this backup altDomain: , // alternate domain from which this app can be reached - xFrameOptions: // set X-Frame-Options header, to control which websites can embed this app + xFrameOptions: , // set X-Frame-Options header, to control which websites can embed this app + robotsTxt: // robots.txt file content } ``` @@ -152,11 +153,14 @@ If `altDomain` is set, the app can be accessed from `https://`. * `SAMEORIGIN` - allows embedding from the same domain as the app. This is the default. * `ALLOW-FROM https://example.com/` - allows this app to be embedded from example.com +Read more about the options at [MDN](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Frame-Options). + `memoryLimit` is the maximum memory this app can use (in bytes) including swap. If set to 0, the app uses the `memoryLimit` value set in the manifest. If set to -1, the app gets unlimited memory. -If `backupId` is provided the app will be initialized with the data from the backup. +If `robotsTxt` if set, it will be returned as the response for `/robots.txt`. You can read about the +[Robots Exclustion Protocol](http://www.robotstxt.org/robotstxt.html) site. -Read more about the options at [MDN](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Frame-Options). +If `backupId` is provided the app will be initialized with the data from the backup. Response (200): @@ -214,7 +218,8 @@ Response (200): }, iconUrl: , // a relative url providing the icon memoryLimit: , // memory constraint in bytes - sso: // Enable single sign-on + sso: , // Enable single sign-on + robotsTxt: // robots.txt file content } ``` @@ -474,7 +479,8 @@ Request: key: , // pem encoded TLS key memoryLimit: , // memory constraint in bytes altDomain: , // alternate domain from which this app can be reached - xFrameOptions: // set X-Frame-Options header, to control which websites can embed this app + xFrameOptions: , // set X-Frame-Options header, to control which websites can embed this app + robotsTxt: // robots.txt file content ``` All values are optional. See [Install app](/references/api.html#install-app) API for field descriptions. diff --git a/migrations/20170714170908-appdb-add-robotsTxt.js b/migrations/20170714170908-appdb-add-robotsTxt.js new file mode 100644 index 000000000..ee05d089c --- /dev/null +++ b/migrations/20170714170908-appdb-add-robotsTxt.js @@ -0,0 +1,15 @@ +'use strict'; + +exports.up = function(db, callback) { + db.runSql('ALTER TABLE apps ADD COLUMN robotsTxt TEXT', function (error) { + if (error) console.error(error); + callback(error); + }); +}; + +exports.down = function(db, callback) { + db.runSql('ALTER TABLE apps DROP COLUMN robotsTxt', function (error) { + if (error) console.error(error); + callback(error); + }); +}; diff --git a/setup/splashpage.sh b/setup/splashpage.sh index a2b6a7c0d..4fc28bbb2 100755 --- a/setup/splashpage.sh +++ b/setup/splashpage.sh @@ -34,11 +34,11 @@ if [[ "${arg_retire_reason}" != "" || "${existing_infra}" != "${current_infra}" echo "Showing progress bar on all subdomains in retired mode or infra update. retire: ${arg_retire_reason} existing: ${existing_infra} current: ${current_infra}" rm -f ${PLATFORM_DATA_DIR}/nginx/applications/* ${box_src_dir}/node_modules/.bin/ejs-cli -f "${script_dir}/start/nginx/appconfig.ejs" \ - -O "{ \"vhost\": \"~^(.+)\$\", \"adminOrigin\": \"${admin_origin}\", \"endpoint\": \"splash\", \"sourceDir\": \"${SETUP_WEBSITE_DIR}\", \"certFilePath\": \"cert/host.cert\", \"keyFilePath\": \"cert/host.key\", \"xFrameOptions\": \"SAMEORIGIN\" }" > "${PLATFORM_DATA_DIR}/nginx/applications/admin.conf" + -O "{ \"vhost\": \"~^(.+)\$\", \"adminOrigin\": \"${admin_origin}\", \"endpoint\": \"splash\", \"sourceDir\": \"${SETUP_WEBSITE_DIR}\", \"certFilePath\": \"cert/host.cert\", \"keyFilePath\": \"cert/host.key\", \"xFrameOptions\": \"SAMEORIGIN\", \"robotsTxt\": null }" > "${PLATFORM_DATA_DIR}/nginx/applications/admin.conf" else echo "Show progress bar only on admin domain for normal update" ${box_src_dir}/node_modules/.bin/ejs-cli -f "${script_dir}/start/nginx/appconfig.ejs" \ - -O "{ \"vhost\": \"${admin_fqdn}\", \"adminOrigin\": \"${admin_origin}\", \"endpoint\": \"splash\", \"sourceDir\": \"${SETUP_WEBSITE_DIR}\", \"certFilePath\": \"cert/host.cert\", \"keyFilePath\": \"cert/host.key\", \"xFrameOptions\": \"SAMEORIGIN\" }" > "${PLATFORM_DATA_DIR}/nginx/applications/admin.conf" + -O "{ \"vhost\": \"${admin_fqdn}\", \"adminOrigin\": \"${admin_origin}\", \"endpoint\": \"splash\", \"sourceDir\": \"${SETUP_WEBSITE_DIR}\", \"certFilePath\": \"cert/host.cert\", \"keyFilePath\": \"cert/host.key\", \"xFrameOptions\": \"SAMEORIGIN\", \"robotsTxt\": null }" > "${PLATFORM_DATA_DIR}/nginx/applications/admin.conf" fi if [[ "${arg_retire_reason}" == "migrate" ]]; then diff --git a/setup/start/nginx/appconfig.ejs b/setup/start/nginx/appconfig.ejs index 51a82ecde..c63167791 100644 --- a/setup/start/nginx/appconfig.ejs +++ b/setup/start/nginx/appconfig.ejs @@ -82,6 +82,12 @@ server { # Disable check to allow unlimited body sizes client_max_body_size 0; +<% if (robotsTxt) { %> + location = /robots.txt { + return 200 "<%= robotsTxt %>"; + } +<% } %> + <% if ( endpoint === 'admin' ) { %> location /api/ { proxy_pass http://127.0.0.1:3000; diff --git a/src/appdb.js b/src/appdb.js index 60e0e8fa0..e7cea43bd 100644 --- a/src/appdb.js +++ b/src/appdb.js @@ -60,7 +60,7 @@ var assert = require('assert'), var APPS_FIELDS_PREFIXED = [ 'apps.id', 'apps.appStoreId', 'apps.installationState', 'apps.installationProgress', 'apps.runState', 'apps.health', 'apps.containerId', 'apps.manifestJson', 'apps.httpPort', 'apps.location', 'apps.dnsRecordId', 'apps.accessRestrictionJson', 'apps.lastBackupId', 'apps.oldConfigJson', 'apps.memoryLimit', 'apps.altDomain', - 'apps.xFrameOptions', 'apps.sso', 'apps.debugModeJson' ].join(','); + 'apps.xFrameOptions', 'apps.sso', 'apps.debugModeJson', 'apps.robotsTxt' ].join(','); var PORT_BINDINGS_FIELDS = [ 'hostPort', 'environmentVariable', 'appId' ].join(','); diff --git a/src/apps.js b/src/apps.js index 65dd4cc9f..93e8c8a62 100644 --- a/src/apps.js +++ b/src/apps.js @@ -243,6 +243,14 @@ function validateDebugMode(debugMode) { return null; } +function validateRobotsTxt(robotsTxt) { + if (robotsTxt === null) return null; + + // TODO: validate the robots file? + + return null; +} + function getDuplicateErrorDetails(location, portBindings, error) { assert.strictEqual(typeof location, 'string'); assert.strictEqual(typeof portBindings, 'object'); @@ -403,6 +411,7 @@ function install(data, auditSource, callback) { xFrameOptions = data.xFrameOptions || 'SAMEORIGIN', sso = 'sso' in data ? data.sso : null, debugMode = data.debugMode || null, + robotsTxt = data.robotsTxt || null, backupId = data.backupId || null; assert(data.appStoreId || data.manifest); // atleast one of them is required @@ -434,6 +443,9 @@ function install(data, auditSource, callback) { error = validateDebugMode(debugMode); if (error) return callback(error); + error = validateRobotsTxt(robotsTxt); + if (error) return callback(error); + if ('sso' in data && !('optionalSso' in manifest)) return callback(new AppsError(AppsError.BAD_FIELD, 'sso can only be specified for apps with optionalSso')); // if sso was unspecified, enable it by default if possible if (sso === null) sso = !!manifest.addons['ldap'] || !!manifest.addons['oauth']; @@ -548,6 +560,12 @@ function configure(appId, data, auditSource, callback) { if (error) return callback(error); } + if ('robotsTxt' in data) { + values.robotsTxt = data.robotsTxt || null; + error = validateRobotsTxt(values.robotsTxt); + if (error) return callback(error); + } + // save cert to boxdata/certs. TODO: move this to apptask when we have a real task queue if ('cert' in data && 'key' in data) { if (data.cert && data.key) { diff --git a/src/nginx.js b/src/nginx.js index 6010ba0c2..ec9af2c25 100644 --- a/src/nginx.js +++ b/src/nginx.js @@ -35,7 +35,8 @@ function configureAdmin(certFilePath, keyFilePath, configFileName, vhost, callba endpoint: 'admin', certFilePath: certFilePath, keyFilePath: keyFilePath, - xFrameOptions: 'SAMEORIGIN' + xFrameOptions: 'SAMEORIGIN', + robotsTxt: 'User-agent: *\\nDisallow: /\\n' }; var nginxConf = ejs.render(NGINX_APPCONFIG_EJS, data); var nginxConfigFilename = path.join(paths.NGINX_APPCONFIG_DIR, configFileName); @@ -63,6 +64,7 @@ function configureApp(app, certFilePath, keyFilePath, callback) { endpoint: endpoint, certFilePath: certFilePath, keyFilePath: keyFilePath, + robotsTxt: app.robotsTxt, xFrameOptions: app.xFrameOptions || 'SAMEORIGIN' // once all apps have been updated/ }; var nginxConf = ejs.render(NGINX_APPCONFIG_EJS, data); diff --git a/src/routes/apps.js b/src/routes/apps.js index 654fb8aff..baa16ec5a 100644 --- a/src/routes/apps.js +++ b/src/routes/apps.js @@ -57,7 +57,8 @@ function removeInternalAppFields(app) { cnameTarget: app.cnameTarget, xFrameOptions: app.xFrameOptions, sso: app.sso, - debugMode: app.debugMode + debugMode: app.debugMode, + robotsTxt: app.robotsTxt }; } @@ -132,6 +133,8 @@ function installApp(req, res, next) { if (('debugMode' in data) && typeof data.debugMode !== 'object') return next(new HttpError(400, 'debugMode must be an object')); + if (data.robotsTxt && typeof data.robotsTxt !== 'string') return next(new HttpError(400, 'robotsTxt must be a string')); + debug('Installing app :%j', data); apps.install(data, auditSource(req), function (error, app) { @@ -170,6 +173,8 @@ function configureApp(req, res, next) { if (('debugMode' in data) && typeof data.debugMode !== 'object') return next(new HttpError(400, 'debugMode must be an object')); + if (data.robotsTxt && typeof data.robotsTxt !== 'string') return next(new HttpError(400, 'robotsTxt must be an object')); + debug('Configuring app id:%s data:%j', req.params.id, data); apps.configure(req.params.id, data, auditSource(req), function (error) { diff --git a/src/test/database-test.js b/src/test/database-test.js index 9a68748e3..905dec2f9 100644 --- a/src/test/database-test.js +++ b/src/test/database-test.js @@ -542,7 +542,8 @@ describe('database', function () { altDomain: null, xFrameOptions: 'DENY', sso: true, - debugMode: null + debugMode: null, + robotsTxt: null }; var APP_1 = { id: 'appid-1', @@ -564,7 +565,8 @@ describe('database', function () { altDomain: null, xFrameOptions: 'SAMEORIGIN', sso: true, - debugMode: null + debugMode: null, + robotsTxt: null }; it('add fails due to missing arguments', function () { diff --git a/webadmin/src/js/client.js b/webadmin/src/js/client.js index abca9c835..e44416195 100644 --- a/webadmin/src/js/client.js +++ b/webadmin/src/js/client.js @@ -336,7 +336,8 @@ angular.module('Application').service('Client', ['$http', 'md5', 'Notification', key: config.key, memoryLimit: config.memoryLimit, altDomain: config.altDomain || null, - xFrameOptions: config.xFrameOptions + xFrameOptions: config.xFrameOptions, + robotsTxt: config.robotsTxt || null }; post('/api/v1/apps/' + id + '/configure', data).success(function (data, status) { diff --git a/webadmin/src/views/apps.html b/webadmin/src/views/apps.html index 37cb386b6..c8c8ee579 100644 --- a/webadmin/src/views/apps.html +++ b/webadmin/src/views/apps.html @@ -144,6 +144,16 @@ +
+ +
+
+ +
+
+
{{ appConfigure.error.cert }}
diff --git a/webadmin/src/views/apps.js b/webadmin/src/views/apps.js index 692f87bfc..aeaabee50 100644 --- a/webadmin/src/views/apps.js +++ b/webadmin/src/views/apps.js @@ -24,6 +24,7 @@ angular.module('Application').controller('AppsController', ['$scope', '$location portBindings: {}, portBindingsEnabled: {}, portBindingsInfo: {}, + robotsEnabled: false, certificateFile: null, certificateFileName: '', keyFile: null, @@ -112,6 +113,7 @@ angular.module('Application').controller('AppsController', ['$scope', '$location $scope.appConfigure.accessRestriction = { users: [], groups: [] }; $scope.appConfigure.xFrameOptions = ''; $scope.appConfigure.customAuth = false; + $scope.appConfigure.robotsEnabled = false; $scope.appConfigureForm.$setPristine(); $scope.appConfigureForm.$setUntouched(); @@ -204,6 +206,7 @@ angular.module('Application').controller('AppsController', ['$scope', '$location $scope.appConfigure.memoryLimit = app.memoryLimit || app.manifest.memoryLimit || (256 * 1024 * 1024); $scope.appConfigure.xFrameOptions = app.xFrameOptions.indexOf('ALLOW-FROM') === 0 ? app.xFrameOptions.split(' ')[1] : ''; $scope.appConfigure.customAuth = !(app.manifest.addons['ldap'] || app.manifest.addons['oauth']); + $scope.appConfigure.robotsEnabled = !!app.robotsTxt; // create ticks starting from manifest memory limit $scope.appConfigure.memoryTicks = [ @@ -256,7 +259,9 @@ angular.module('Application').controller('AppsController', ['$scope', '$location cert: $scope.appConfigure.certificateFile, key: $scope.appConfigure.keyFile, xFrameOptions: $scope.appConfigure.xFrameOptions ? ('ALLOW-FROM ' + $scope.appConfigure.xFrameOptions) : 'SAMEORIGIN', - memoryLimit: $scope.appConfigure.memoryLimit === $scope.appConfigure.memoryTicks[0] ? 0 : $scope.appConfigure.memoryLimit + memoryLimit: $scope.appConfigure.memoryLimit === $scope.appConfigure.memoryTicks[0] ? 0 : $scope.appConfigure.memoryLimit, + // preserve arbitrary robots.txt in database + robotsTxt: $scope.appConfigure.robotsEnabled ? ($scope.appConfigure.app.robotsTxt || 'User-agent: *\\nDisallow: /\\n') : null }; Client.configureApp($scope.appConfigure.app.id, $scope.appConfigure.password, data, function (error) {