diff --git a/src/apps.js b/src/apps.js index 8fa5d9e68..ffddc03db 100644 --- a/src/apps.js +++ b/src/apps.js @@ -137,6 +137,7 @@ const appdb = require('./appdb.js'), reverseProxy = require('./reverseproxy.js'), safe = require('safetydance'), semver = require('semver'), + services = require('./services.js'), settings = require('./settings.js'), spawn = require('child_process').spawn, split = require('split'), @@ -1273,7 +1274,10 @@ function setDataDir(app, dataDir, auditSource, callback) { const task = { args: { newDataDir: dataDir }, - values: { } + values: { }, + onFinished: (error) => { + if (!error) services.rebuildService('sftp', NOOP_CALLBACK); + } }; addTask(appId, exports.ISTATE_PENDING_DATA_DIR_MIGRATION, task, function (error, result) { if (error) return callback(error); diff --git a/src/apptaskmanager.js b/src/apptaskmanager.js index 74eb9e884..45afba9fe 100644 --- a/src/apptaskmanager.js +++ b/src/apptaskmanager.js @@ -4,7 +4,7 @@ exports = module.exports = { scheduleTask }; -let assert = require('assert'), +const assert = require('assert'), BoxError = require('./boxerror.js'), debug = require('debug')('box:apptaskmanager'), fs = require('fs'), @@ -13,7 +13,6 @@ let assert = require('assert'), path = require('path'), paths = require('./paths.js'), scheduler = require('./scheduler.js'), - services = require('./services.js'), tasks = require('./tasks.js'); let gActiveTasks = { }; // indexed by app id @@ -84,15 +83,10 @@ function scheduleTask(appId, taskId, options, callback) { } function startNextTask() { - if (gPendingTasks.length === 0) { - // rebuild sftp when task queue is empty. this minimizes risk of sftp rebuild overlapping with other app tasks - services.rebuildService('sftp', error => { if (error) debug('Unable to rebuild sftp:', error); }); - return; - } + if (gPendingTasks.length === 0) return; assert(Object.keys(gActiveTasks).length < TASK_CONCURRENCY); const t = gPendingTasks.shift(); scheduleTask(t.appId, t.taskId, t.options, t.callback); } - diff --git a/src/infra_version.js b/src/infra_version.js index 53bca7aee..337dcaca1 100644 --- a/src/infra_version.js +++ b/src/infra_version.js @@ -22,6 +22,6 @@ exports = module.exports = { 'redis': { repo: 'cloudron/redis', tag: 'cloudron/redis:3.0.3@sha256:37e5222e01ae89bc5a742ce12030631de25a127b5deec8a0e992c68df0fdec10' }, 'mail': { repo: 'cloudron/mail', tag: 'cloudron/mail:3.3.2@sha256:d0d153612f478a0ef099809d4c3c72c3e02f43a55a796987d922f16367e7881e' }, 'graphite': { repo: 'cloudron/graphite', tag: 'cloudron/graphite:3.0.1@sha256:bed9f6b5d06fe2c5289e895e806cfa5b74ad62993d705be55d4554a67d128029' }, - 'sftp': { repo: 'cloudron/sftp', tag: 'cloudron/sftp:3.2.0@sha256:61e8247ded1e07cf882ca478dab180960357c614472e80b938f1f690a46788c2' } + 'sftp': { repo: 'cloudron/sftp', tag: 'cloudron/sftp:3.3.0@sha256:183c11150d5a681cb02f7d2bd542ddb8a8f097422feafb7fac8fdbca0ca55d47' } } }; diff --git a/src/ldap.js b/src/ldap.js index f03cf02a1..a10f243c4 100644 --- a/src/ldap.js +++ b/src/ldap.js @@ -17,7 +17,6 @@ const assert = require('assert'), ldap = require('ldapjs'), mail = require('./mail.js'), mailboxdb = require('./mailboxdb.js'), - path = require('path'), safe = require('safetydance'), services = require('./services.js'), users = require('./users.js'); @@ -625,7 +624,7 @@ function userSearchSftp(req, res, next) { var obj = { dn: ldap.parseDN(`cn=${username}@${appFqdn},ou=sftp,dc=cloudron`).toString(), attributes: { - homeDirectory: path.join('/app/data', app.id), + homeDirectory: app.dataDir ? `/mnt/${app.id}` : `/mnt/appsdata/${app.id}/data`, objectclass: ['user'], objectcategory: 'person', cn: user.id, diff --git a/src/sftp.js b/src/sftp.js index 0c7bf0b42..1cdcd5852 100644 --- a/src/sftp.js +++ b/src/sftp.js @@ -10,16 +10,15 @@ exports = module.exports = { const apps = require('./apps.js'), assert = require('assert'), async = require('async'), + BoxError = require('./boxerror.js'), debug = require('debug')('box:sftp'), - docker = require('./docker.js'), hat = require('./hat.js'), infra = require('./infra_version.js'), paths = require('./paths.js'), safe = require('safetydance'), shell = require('./shell.js'), system = require('./system.js'), - volumes = require('./volumes.js'), - _ = require('underscore'); + volumes = require('./volumes.js'); function rebuild(serviceConfig, options, callback) { assert.strictEqual(typeof serviceConfig, 'object'); @@ -28,20 +27,27 @@ function rebuild(serviceConfig, options, callback) { debug('rebuilding container'); - const force = !!options.force; const tag = infra.images.sftp.tag; const memoryLimit = serviceConfig.memoryLimit || exports.DEFAULT_MEMORY_LIMIT; const memory = system.getMemoryAllocation(memoryLimit); const cloudronToken = hat(8 * 128); + let dataDirs = []; + + const stat = safe.fs.lstatSync(paths.APPS_DATA_DIR); + if (!stat) return callback(new BoxError(BoxError.FS_ERROR, safe.error)); + + const resolvedAppDataDir = stat.isSymbolicLink() ? safe.fs.readlinkSync(paths.APPS_DATA_DIR) : paths.APPS_DATA_DIR; + + dataDirs.push({ hostDir: resolvedAppDataDir, mountDir: '/mnt/appsdata' }); + apps.getAll(async function (error, result) { if (error) return callback(error); - let dataDirs = []; result.forEach(function (app) { - if (!app.manifest.addons['localstorage']) return; + if (!app.manifest.addons['localstorage'] || !app.dataDir) return; - const hostDir = apps.getDataDir(app, app.dataDir), mountDir = `/app/data/${app.id}`; + const hostDir = apps.getDataDir(app, app.dataDir), mountDir = `/mnt/${app.id}`; if (!safe.fs.existsSync(hostDir)) { // this can fail if external mount does not have permissions for yellowtent user // do not create host path when cloudron is restoring. this will then create dir with root perms making restore logic fail debug(`Ignoring app data dir ${hostDir} for ${app.id} since it does not exist`); @@ -55,59 +61,45 @@ function rebuild(serviceConfig, options, callback) { [error, allVolumes] = await safe(volumes.list()); if (error) return callback(error); + dataDirs.push({ hostDir: '/mnt/volumes', mountDir: '/mnt/volumes' }); + allVolumes.forEach(function (volume) { + if (volume.hostPath.startsWith('/mnt/volumes/')) return; + if (!safe.fs.existsSync(volume.hostPath)) { debug(`Ignoring volume host path ${volume.hostPath} since it does not exist`); return; } - dataDirs.push({ hostDir: volume.hostPath, mountDir: `/app/data/${volume.id}` }); + dataDirs.push({ hostDir: volume.hostPath, mountDir: `/mnt/${volume.id}` }); }); - docker.inspect('sftp', function (error, data) { - if (!error && data && data.Mounts) { - let currentDataDirs = data.Mounts; - if (currentDataDirs) { - currentDataDirs = currentDataDirs.filter(function (d) { return d.Destination.indexOf('/app/data/') === 0; }).map(function (d) { return { hostDir: d.Source, mountDir: d.Destination }; }); + const mounts = dataDirs.map(function (v) { return `-v "${v.hostDir}:${v.mountDir}"`; }).join(' '); + const cmd = `docker run --restart=always -d --name="sftp" \ + --hostname sftp \ + --net cloudron \ + --net-alias sftp \ + --log-driver syslog \ + --log-opt syslog-address=udp://127.0.0.1:2514 \ + --log-opt syslog-format=rfc5424 \ + --log-opt tag=sftp \ + -m ${memory} \ + --memory-swap ${memoryLimit} \ + --dns 172.18.0.1 \ + --dns-search=. \ + -p 222:22 \ + ${mounts} \ + -e CLOUDRON_SFTP_TOKEN="${cloudronToken}" \ + -v "${paths.SFTP_KEYS_DIR}:/etc/ssh:ro" \ + --label isCloudronManaged=true \ + --read-only -v /tmp -v /run "${tag}"`; - // sort for comparison - currentDataDirs.sort(function (a, b) { return a.hostDir < b.hostDir ? -1 : 1; }); - dataDirs.sort(function (a, b) { return a.hostDir < b.hostDir ? -1 : 1; }); - - if (!force && _.isEqual(currentDataDirs, dataDirs)) { - debug('Skipping rebuild, no changes'); - return callback(); - } - } - } - - const mounts = dataDirs.map(function (v) { return `-v "${v.hostDir}:${v.mountDir}"`; }).join(' '); - const cmd = `docker run --restart=always -d --name="sftp" \ - --hostname sftp \ - --net cloudron \ - --net-alias sftp \ - --log-driver syslog \ - --log-opt syslog-address=udp://127.0.0.1:2514 \ - --log-opt syslog-format=rfc5424 \ - --log-opt tag=sftp \ - -m ${memory} \ - --memory-swap ${memoryLimit} \ - --dns 172.18.0.1 \ - --dns-search=. \ - -p 222:22 \ - ${mounts} \ - -e CLOUDRON_SFTP_TOKEN="${cloudronToken}" \ - -v "${paths.SFTP_KEYS_DIR}:/etc/ssh:ro" \ - --label isCloudronManaged=true \ - --read-only -v /tmp -v /run "${tag}"`; - - // ignore error if container not found (and fail later) so that this code works across restarts - async.series([ - shell.exec.bind(null, 'stopSftp', 'docker stop sftp || true'), - shell.exec.bind(null, 'removeSftp', 'docker rm -f sftp || true'), - shell.exec.bind(null, 'startSftp', cmd) - ], callback); - }); + // ignore error if container not found (and fail later) so that this code works across restarts + async.series([ + shell.exec.bind(null, 'stopSftp', 'docker stop sftp || true'), + shell.exec.bind(null, 'removeSftp', 'docker rm -f sftp || true'), + shell.exec.bind(null, 'startSftp', cmd) + ], callback); }); } diff --git a/src/volumes.js b/src/volumes.js index 4540322a8..ed581c0d3 100644 --- a/src/volumes.js +++ b/src/volumes.js @@ -98,7 +98,7 @@ async function add(volume, auditSource) { } eventlog.add(eventlog.ACTION_VOLUME_ADD, auditSource, { id, name, hostPath }); - services.rebuildService('sftp', NOOP_CALLBACK); + if (mountType === 'noop') services.rebuildService('sftp', NOOP_CALLBACK); const collectdConf = ejs.render(COLLECTD_CONFIG_EJS, { volumeId: id, hostPath }); collectd.addProfile(id, collectdConf, NOOP_CALLBACK); @@ -152,8 +152,12 @@ async function del(volume, auditSource) { } eventlog.add(eventlog.ACTION_VOLUME_REMOVE, auditSource, { volume }); - services.rebuildService('sftp', async function () { - if (volume.mountType !== 'noop') await safe(mounts.removeMount(volume)); - }); + + if (volume.mountType === 'noop') { + services.rebuildService('sftp', NOOP_CALLBACK); + } else { + await safe(mounts.removeMount(volume)); + } + collectd.removeProfile(volume.id, NOOP_CALLBACK); }