diff --git a/runTests b/runTests index 0303cd5c0..28814dadb 100755 --- a/runTests +++ b/runTests @@ -22,7 +22,7 @@ fi mkdir -p ${DATA_DIR} cd ${DATA_DIR} mkdir -p appsdata -mkdir -p boxdata/mail boxdata/certs boxdata/mail/dkim/localhost boxdata/mail/dkim/foobar.com +mkdir -p boxdata/box boxdata/mail boxdata/certs boxdata/mail/dkim/localhost boxdata/mail/dkim/foobar.com mkdir -p platformdata/addons/mail/banner platformdata/nginx/cert platformdata/nginx/applications platformdata/collectd/collectd.conf.d platformdata/addons platformdata/logrotate.d platformdata/backup platformdata/logs/tasks platformdata/sftp/ssh platformdata/firewall platformdata/update sudo mkdir -p /mnt/cloudron-test-music /media/cloudron-test-music # volume test diff --git a/setup/start.sh b/setup/start.sh index c29b1a20b..f35260b88 100755 --- a/setup/start.sh +++ b/setup/start.sh @@ -16,7 +16,7 @@ readonly HOME_DIR="/home/${USER}" readonly BOX_SRC_DIR="${HOME_DIR}/box" readonly PLATFORM_DATA_DIR="${HOME_DIR}/platformdata" readonly APPS_DATA_DIR="${HOME_DIR}/appsdata" -readonly BOX_DATA_DIR="${HOME_DIR}/boxdata" +readonly BOX_DATA_DIR="${HOME_DIR}/boxdata/box" readonly MAIL_DATA_DIR="${HOME_DIR}/boxdata/mail" readonly script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -229,9 +229,8 @@ chown "${USER}:${USER}" "${PLATFORM_DATA_DIR}/INFRA_VERSION" 2>/dev/null || true chown "${USER}:${USER}" "${PLATFORM_DATA_DIR}" chown "${USER}:${USER}" "${APPS_DATA_DIR}" -# do not chown the boxdata/mail directory; dovecot gets upset -chown "${USER}:${USER}" "${BOX_DATA_DIR}" -find "${BOX_DATA_DIR}" -mindepth 1 -maxdepth 1 -not -path "${MAIL_DATA_DIR}" -exec chown -R "${USER}:${USER}" {} \; +chown "${USER}:${USER}" -R "${BOX_DATA_DIR}" +# do not chown the boxdata/mail directory entirely; dovecot gets upset chown "${USER}:${USER}" "${MAIL_DATA_DIR}" chown "${USER}:${USER}" -R "${MAIL_DATA_DIR}/dkim" # this is owned by box currently since it generates the keys diff --git a/src/backupcleaner.js b/src/backupcleaner.js index 92f883834..8077e5099 100644 --- a/src/backupcleaner.js +++ b/src/backupcleaner.js @@ -261,7 +261,7 @@ async function run(progressCallback) { } await progressCallback({ percent: 10, message: 'Cleaning box backups' }); - const { removedBoxBackupIds, referencedAppBackupIds } = await cleanupBoxBackups(backupConfig, progressCallback);; + const { removedBoxBackupIds, referencedAppBackupIds } = await cleanupBoxBackups(backupConfig, progressCallback); await progressCallback({ percent: 40, message: 'Cleaning app backups' }); const removedAppBackupIds = await cleanupAppBackups(backupConfig, referencedAppBackupIds, progressCallback); diff --git a/src/backups.js b/src/backups.js index 19ed80440..21c883018 100644 --- a/src/backups.js +++ b/src/backups.js @@ -28,9 +28,11 @@ exports = module.exports = { testProviderConfig, BACKUP_IDENTIFIER_BOX: 'box', + BACKUP_IDENTIFIER_MAIL: 'mail', BACKUP_TYPE_APP: 'app', BACKUP_TYPE_BOX: 'box', + BACKUP_TYPE_MAIL: 'mail', BACKUP_STATE_NORMAL: 'normal', // should rename to created to avoid listing in UI? BACKUP_STATE_CREATING: 'creating', @@ -238,6 +240,7 @@ function getSnapshotInfo(id) { return info[id] || { }; } +// keeps track of contents of the snapshot directory. this provides a way to clean up backups of uninstalled apps async function setSnapshotInfo(id, info) { assert.strictEqual(typeof id, 'string'); assert.strictEqual(typeof info, 'object'); diff --git a/src/backuptask.js b/src/backuptask.js index c107e1ef9..8df3e835a 100644 --- a/src/backuptask.js +++ b/src/backuptask.js @@ -1,13 +1,16 @@ 'use strict'; exports = module.exports = { - backupBoxAndApps, + fullBackup, restore, backupApp, downloadApp, + backupMail, + downloadMail, + upload, _restoreFsMetadata: restoreFsMetadata, @@ -750,11 +753,39 @@ async function uploadBoxSnapshot(backupConfig, progressCallback) { await backups.setSnapshotInfo('box', { timestamp: new Date().toISOString(), format: backupConfig.format }); } -async function rotateBoxBackup(backupConfig, tag, options, appBackupIds, progressCallback) { +async function copy(backupConfig, sourceBackupId, destBackupId, options, progressCallback) { + assert.strictEqual(typeof backupConfig, 'object'); + assert.strictEqual(typeof sourceBackupId, 'string'); + assert.strictEqual(typeof destBackupId, 'string'); + assert.strictEqual(typeof options, 'object'); + assert.strictEqual(typeof progressCallback, 'function'); + + const format = backupConfig.format; + + return new Promise((resolve, reject) => { + const startTime = new Date(); + + const copyEvents = storage.api(backupConfig.provider).copy(backupConfig, storage.getBackupFilePath(backupConfig, sourceBackupId, format), storage.getBackupFilePath(backupConfig, destBackupId, format)); + copyEvents.on('progress', (message) => progressCallback({ message })); + copyEvents.on('done', async function (copyBackupError) { + const state = copyBackupError ? backups.BACKUP_STATE_ERROR : backups.BACKUP_STATE_NORMAL; + + const [error] = await safe(backups.update(destBackupId, { preserveSecs: options.preserveSecs || 0, state })); + if (copyBackupError) return reject(copyBackupError); + if (error) return reject(error); + + debug(`copy: copied successfully to id ${destBackupId}. Took ${(new Date() - startTime)/1000} seconds`); + + resolve(); + }); + }); +} + +async function rotateBoxBackup(backupConfig, tag, options, dependsOn, progressCallback) { assert.strictEqual(typeof backupConfig, 'object'); assert.strictEqual(typeof tag, 'string'); assert.strictEqual(typeof options, 'object'); - assert(Array.isArray(appBackupIds)); + assert(Array.isArray(dependsOn)); assert.strictEqual(typeof progressCallback, 'function'); const backupId = `${tag}/box_v${constants.VERSION}`; @@ -767,33 +798,19 @@ async function rotateBoxBackup(backupConfig, tag, options, appBackupIds, progres packageVersion: constants.VERSION, type: backups.BACKUP_TYPE_BOX, state: backups.BACKUP_STATE_CREATING, - identifier: 'box', - dependsOn: appBackupIds, + identifier: backups.BACKUP_IDENTIFIER_BOX, + dependsOn, manifest: null, - format: format + format }; await backups.add(backupId, data); - - return new Promise((resolve, reject) => { - const copy = storage.api(backupConfig.provider).copy(backupConfig, storage.getBackupFilePath(backupConfig, 'snapshot/box', format), storage.getBackupFilePath(backupConfig, backupId, format)); - copy.on('progress', (message) => progressCallback({ message: `box: ${message}` })); - copy.on('done', async function (copyBackupError) { - const state = copyBackupError ? backups.BACKUP_STATE_ERROR : backups.BACKUP_STATE_NORMAL; - - const [error] = await safe(backups.update(backupId, { preserveSecs: options.preserveSecs || 0, state })); - if (copyBackupError) return reject(copyBackupError); - if (error) return reject(error); - - debug(`rotateBoxBackup: rotated successfully as id ${backupId}`); - - resolve(backupId); - }); - }); + await copy(backupConfig, 'snapshot/box', backupId, options, progressCallback); + return backupId; } -async function backupBoxWithAppBackupIds(appBackupIds, tag, options, progressCallback) { - assert(Array.isArray(appBackupIds)); +async function backupBox(dependsOn, tag, options, progressCallback) { + assert(Array.isArray(dependsOn)); assert.strictEqual(typeof tag, 'string'); assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof progressCallback, 'function'); @@ -802,7 +819,7 @@ async function backupBoxWithAppBackupIds(appBackupIds, tag, options, progressCal await uploadBoxSnapshot(backupConfig, progressCallback); - const backupId = await rotateBoxBackup(backupConfig, tag, options, appBackupIds, progressCallback); + const backupId = await rotateBoxBackup(backupConfig, tag, options, dependsOn, progressCallback); return backupId; } @@ -813,8 +830,6 @@ async function rotateAppBackup(backupConfig, app, tag, options, progressCallback assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof progressCallback, 'function'); - const startTime = new Date(); - const snapshotInfo = backups.getSnapshotInfo(app.id); const manifest = snapshotInfo.restoreConfig ? snapshotInfo.restoreConfig.manifest : snapshotInfo.manifest; // compat @@ -835,22 +850,8 @@ async function rotateAppBackup(backupConfig, app, tag, options, progressCallback }; await backups.add(backupId, data); - - return new Promise((resolve, reject) => { - const copy = storage.api(backupConfig.provider).copy(backupConfig, storage.getBackupFilePath(backupConfig, `snapshot/app_${app.id}`, format), storage.getBackupFilePath(backupConfig, backupId, format)); - copy.on('progress', (message) => progressCallback({ message: `${message} (${app.fqdn})` })); - copy.on('done', async function (copyBackupError) { - const state = copyBackupError ? backups.BACKUP_STATE_ERROR : backups.BACKUP_STATE_NORMAL; - - const [error] = await safe(backups.update(backupId, { preserveSecs: options.preserveSecs || 0, state })); - if (copyBackupError) return reject(copyBackupError); - if (error) return reject(error); - - debug(`rotateAppBackup: rotated ${app.fqdn} to id ${backupId}. Took ${(new Date() - startTime)/1000} seconds`); - - resolve(backupId); - }); - }); + await copy(backupConfig, `snapshot/app_${app.id}`, backupId, options, progressCallback); + return backupId; } async function backupApp(app, options, progressCallback) { @@ -931,8 +932,98 @@ async function backupAppWithTag(app, tag, options, progressCallback) { return backupId; } -// this function expects you to have a lock. Unlike other progressCallback this also has a progress field -async function backupBoxAndApps(options, progressCallback) { +async function uploadMailSnapshot(backupConfig, progressCallback) { + assert.strictEqual(typeof backupConfig, 'object'); + assert.strictEqual(typeof progressCallback, 'function'); + + const mailDataDir = safe.fs.realpathSync(paths.MAIL_DATA_DIR); + if (!mailDataDir) throw new BoxError(BoxError.FS_ERROR, `Error resolving maildata: ${safe.error.message}`); + + const uploadConfig = { + backupId: 'snapshot/mail', + backupConfig, + dataLayout: new DataLayout(mailDataDir, []), + progressTag: 'mail' + }; + + progressCallback({ message: 'Uploading mail snapshot' }); + + const startTime = new Date(); + + await util.promisify(runBackupUpload)(uploadConfig, progressCallback); + + debug(`uploadMailSnapshot: took ${(new Date() - startTime)/1000} seconds`); + + await backups.setSnapshotInfo('mail', { timestamp: new Date().toISOString(), format: backupConfig.format }); +} + +async function rotateMailBackup(backupConfig, tag, options, progressCallback) { + assert.strictEqual(typeof backupConfig, 'object'); + assert.strictEqual(typeof tag, 'string'); + assert.strictEqual(typeof options, 'object'); + assert.strictEqual(typeof progressCallback, 'function'); + + const backupId = `${tag}/mail_v${constants.VERSION}`; + const format = backupConfig.format; + + debug(`rotateMailBackup: rotating to id ${backupId}`); + + const data = { + encryptionVersion: backupConfig.encryption ? 2 : null, + packageVersion: constants.VERSION, + type: backups.BACKUP_TYPE_MAIL, + state: backups.BACKUP_STATE_CREATING, + identifier: backups.BACKUP_IDENTIFIER_MAIL, + dependsOn: [], + manifest: null, + format: format + }; + + await backups.add(backupId, data); + await copy(backupConfig, 'snapshot/mail', backupId, options, progressCallback); + return backupId; +} + +async function backupMailWithTag(tag, options, progressCallback) { + assert.strictEqual(typeof tag, 'string'); + assert.strictEqual(typeof options, 'object'); + assert.strictEqual(typeof progressCallback, 'function'); + + debug(`backupMailWithTag: backing up mail with tag ${tag}`); + + const backupConfig = await settings.getBackupConfig(); + await uploadMailSnapshot(backupConfig, progressCallback); + const backupId = await rotateMailBackup(backupConfig, tag, options, progressCallback); + return backupId; +} + +async function backupMail(options, progressCallback) { + assert.strictEqual(typeof options, 'object'); + assert.strictEqual(typeof progressCallback, 'function'); + + const tag = (new Date()).toISOString().replace(/[T.]/g, '-').replace(/[:Z]/g,''); + debug(`backupMail: backing up mail with tag ${tag}`); + return await backupMailWithTag(tag, options, progressCallback); +} + +async function downloadMail(restoreConfig, progressCallback) { + assert.strictEqual(typeof restoreConfig, 'object'); + assert.strictEqual(typeof progressCallback, 'function'); + + const mailDataDir = safe.fs.realpathSync(paths.MAIL_DATA_DIR); + if (!mailDataDir) throw new BoxError(BoxError.FS_ERROR, `Error resolving maildata: ${safe.error.message}`); + const dataLayout = new DataLayout(mailDataDir, []); + + const startTime = new Date(); + const backupConfig = restoreConfig.backupConfig || await settings.getBackupConfig(); + + const downloadAsync = util.promisify(download); + await downloadAsync(backupConfig, restoreConfig.backupId, restoreConfig.backupFormat, dataLayout, progressCallback); + debug('downloadMail: time: %s', (new Date() - startTime)/1000); +} + +// this function is called from external process. calling process is expected to have a lock +async function fullBackup(options, progressCallback) { assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof progressCallback, 'function'); @@ -941,7 +1032,7 @@ async function backupBoxAndApps(options, progressCallback) { const allApps = await apps.list(); let percent = 1; - let step = 100/(allApps.length+2); + let step = 100/(allApps.length+3); const appBackupIds = []; for (const app of allApps) { @@ -949,19 +1040,24 @@ async function backupBoxAndApps(options, progressCallback) { percent += step; if (!app.enableBackup) { - debug(`Skipped backup ${app.fqdn}`); + debug(`fullBackup: skipped backup ${app.fqdn}`); return; // nothing to backup } const startTime = new Date(); - const backupId = await backupAppWithTag(app, tag, options, (progress) => progressCallback({ percent: percent, message: progress.message })); - debug(`backupBoxAndApps: app ${app.fqdn} backup finished. Took ${(new Date() - startTime)/1000} seconds`); - if (backupId) appBackupIds.push(backupId); // backupId can be null if in BAD_STATE and never backed up + const appBackupId = await backupAppWithTag(app, tag, options, (progress) => progressCallback({ percent: percent, message: progress.message })); + debug(`fullBackup: app ${app.fqdn} backup finished. Took ${(new Date() - startTime)/1000} seconds`); + if (appBackupId) appBackupIds.push(appBackupId); // backupId can be null if in BAD_STATE and never backed up } + progressCallback({ percent: percent, message: 'Backing up mail' }); + percent += step; + const mailBackupId = await backupMailWithTag(tag, options, (progress) => progressCallback({ percent: percent, message: progress.message })); + progressCallback({ percent: percent, message: 'Backing up system data' }); percent += step; - const backupId = await backupBoxWithAppBackupIds(appBackupIds, tag, options, (progress) => progressCallback({ percent: percent, message: progress.message })); + const dependsOn = appBackupIds.concat(mailBackupId); + const backupId = await backupBox(dependsOn, tag, options, (progress) => progressCallback({ percent: percent, message: progress.message })); return backupId; } diff --git a/src/paths.js b/src/paths.js index 43b6046a8..bb2410c55 100644 --- a/src/paths.js +++ b/src/paths.js @@ -25,7 +25,6 @@ exports = module.exports = { PLATFORM_DATA_DIR: path.join(baseDir(), 'platformdata'), APPS_DATA_DIR: path.join(baseDir(), 'appsdata'), - BOX_DATA_DIR: path.join(baseDir(), 'boxdata'), // box data dir is part of box backup ACME_CHALLENGES_DIR: path.join(baseDir(), 'platformdata/acme'), ADDON_CONFIG_DIR: path.join(baseDir(), 'platformdata/addons'), @@ -49,6 +48,7 @@ exports = module.exports = { SFTP_PRIVATE_KEY_FILE: path.join(baseDir(), 'platformdata/sftp/ssh/ssh_host_rsa_key'), FIREWALL_BLOCKLIST_FILE: path.join(baseDir(), 'platformdata/firewall/blocklist.txt'), + BOX_DATA_DIR: path.join(baseDir(), 'boxdata/box'), MAIL_DATA_DIR: path.join(baseDir(), 'boxdata/mail'), LOG_DIR: path.join(baseDir(), 'platformdata/logs'), diff --git a/src/system.js b/src/system.js index cbcb684e0..6f95bf860 100644 --- a/src/system.js +++ b/src/system.js @@ -71,7 +71,7 @@ async function getDisks() { const ext4Disks = allDisks.filter((r) => r.type === 'ext4').sort((a, b) => a.mountpoint.localeCompare(b.mountpoint)); const diskInfos = []; - for (const p of [ paths.BOX_DATA_DIR, paths.PLATFORM_DATA_DIR, paths.APPS_DATA_DIR, info.DockerRootDir ]) { + for (const p of [ paths.BOX_DATA_DIR, paths.MAIL_DATA_DIR, paths.PLATFORM_DATA_DIR, paths.APPS_DATA_DIR, info.DockerRootDir ]) { const [dfError, diskInfo] = await safe(df.file(p)); if (dfError) throw new BoxError(BoxError.FS_ERROR, dfError); diskInfos.push(diskInfo); @@ -82,10 +82,10 @@ async function getDisks() { const result = { disks: ext4Disks, // root disk is first. { filesystem, type, size, used, avialable, capacity, mountpoint } boxDataDisk: diskInfos[0].filesystem, - mailDataDisk: diskInfos[0].filesystem, - platformDataDisk: diskInfos[1].filesystem, - appsDataDisk: diskInfos[2].filesystem, - dockerDataDisk: diskInfos[3].filesystem, + mailDataDisk: diskInfos[1].filesystem, + platformDataDisk: diskInfos[2].filesystem, + appsDataDisk: diskInfos[3].filesystem, + dockerDataDisk: diskInfos[4].filesystem, backupsDisk: backupsFilesystem, apps: {}, // filled below volumes: {} // filled below diff --git a/src/taskworker.js b/src/taskworker.js index 58866dda6..b7aec1b8b 100755 --- a/src/taskworker.js +++ b/src/taskworker.js @@ -21,7 +21,7 @@ const apptask = require('./apptask.js'), const TASKS = { // indexed by task type app: apptask.run, - backup: backuptask.backupBoxAndApps, + backup: backuptask.fullBackup, update: updater.update, checkCerts: reverseProxy.checkCerts, setupDnsAndCert: cloudron.setupDnsAndCert, diff --git a/src/test/backuptask-test.js b/src/test/backuptask-test.js index a4bc0e98a..d5b9bfb83 100644 --- a/src/test/backuptask-test.js +++ b/src/test/backuptask-test.js @@ -64,7 +64,7 @@ describe('backuptask', function () { }); - describe('backupBoxAndApps', function () { + describe('fullBackup', function () { let backupInfo1; const backupConfig = { diff --git a/src/updater.js b/src/updater.js index bbc7bab99..172d3142a 100644 --- a/src/updater.js +++ b/src/updater.js @@ -25,8 +25,7 @@ const apps = require('./apps.js'), settings = require('./settings.js'), shell = require('./shell.js'), tasks = require('./tasks.js'), - updateChecker = require('./updatechecker.js'), - util = require('util'); + updateChecker = require('./updatechecker.js'); const RELEASES_PUBLIC_KEY = path.join(__dirname, 'releases.gpg'); const UPDATE_CMD = path.join(__dirname, 'scripts/update.sh'); @@ -148,7 +147,7 @@ async function update(boxUpdateInfo, options, progressCallback) { if (!options.skipBackup) { progressCallback({ percent: 10, message: 'Backing up' }); - await backuptask.backupBoxAndApps({ preserveSecs: 3*7*24*60*60 }, (progress) => progressCallback({ percent: 10+progress.percent*70/100, message: progress.message })); + await backuptask.fullBackup({ preserveSecs: 3*7*24*60*60 }, (progress) => progressCallback({ percent: 10+progress.percent*70/100, message: progress.message })); } debug('updating box %s', boxUpdateInfo.sourceTarballUrl);