diff --git a/migrations/20250724102340-backupTargets-create-table.js b/migrations/20250724102340-backupTargets-create-table.js index 098be84ed..965052fb7 100644 --- a/migrations/20250724102340-backupTargets-create-table.js +++ b/migrations/20250724102340-backupTargets-create-table.js @@ -34,21 +34,36 @@ exports.up = async function (db) { if (results.length === 0) { provider = 'filesystem'; - config = { id, _provider: provider, backupFolder: paths.DEFAULT_BACKUP_DIR }; + config = { id, _provider: provider, backupDir: paths.DEFAULT_BACKUP_DIR }; format = 'tgz'; } else { for (const r of results) { if (r.name === 'backup_storage') { const tmp = JSON.parse(r.value); + // provider is top level provider = tmp.provider; + // the s3 and filesystem backend use the _provider internal property + if (provider !== 'gcs' && provider !== 'noop') tmp._provider = tmp.provider; + delete tmp.provider; + + // backupFolder is now backupDir + if ('backupFolder' in tmp) { + tmp.backupDir = tmp.backupFolder; + delete tmp.backupFolder; + } + + // encryption is not part of config anymore, it is top level encryption = tmp.encryption || null; delete tmp.encryption; + // format is not part of config anymore, it is top level format = tmp.format; delete tmp.format; - tmp._managedMountPath = '/mnt/cloudronbackup'; + // previous releases only had a single "managed" mount at /mnt/cloudronbackup . + // new release has it under /mnt/managedbackups . + if (tmp.mountOptions) tmp._managedMountPath = '/mnt/cloudronbackup'; config = tmp; } else if (r.name === 'backup_limits') { diff --git a/migrations/20250724141339-backups-add-targetId.js b/migrations/20250724141339-backups-add-targetId.js index 377a18935..ad74f6b50 100644 --- a/migrations/20250724141339-backups-add-targetId.js +++ b/migrations/20250724141339-backups-add-targetId.js @@ -1,6 +1,8 @@ 'use strict'; -const crypto = require('crypto'); +const crypto = require('crypto'), + path = require('path'), + paths = require('../src/paths.js'); exports.up = async function(db) { let results = await db.runSql('SELECT format, COUNT(*) AS count FROM backups GROUP BY format WITH ROLLUP', []); // https://dev.mysql.com/doc/refman/8.4/en/group-by-modifiers.html @@ -24,7 +26,7 @@ exports.up = async function(db) { cloneBackupTarget.format = currentBackupTarget.format === 'rsync' ? 'tgz' : 'rsync'; cloneBackupTarget.priority = false; cloneBackupTarget.schedule = 'never'; - cloneBackupTarget._managedMountPath = `/mnt/backups/${cloneId}`; // this won't work until the user remounts + cloneBackupTarget._managedMountPath = path.join(paths.MANAGED_BACKUP_MOUNT_DIR, cloneId); // this won't work until the user remounts console.log(`Existing format is ${currentBackupTarget.format} . Adding clone backup target for ${cloneBackupTarget.format}`); await db.runSql('INSERT INTO backupTargets (id, label, configJson, limitsJson, retentionJson, schedule, encryptionJson, format, priority) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)', diff --git a/src/backuptargets.js b/src/backuptargets.js index 18e735640..8dbae6f26 100644 --- a/src/backuptargets.js +++ b/src/backuptargets.js @@ -57,7 +57,7 @@ const assert = require('assert'), // gcs - bucket, prefix, projectId, credentials . see gcs.js // ext4/xfs/disk (managed providers) - mountOptions (diskPath), prefix, noHardlinks. disk is legacy. // nfs/cifs/sshfs (managed providers) - mountOptions (host/username/password/seal/privateKey etc), prefix, noHardlinks -// filesystem - backupFolder, noHardlinks +// filesystem - backupDir, noHardlinks // mountpoint - mountPoint, prefix, noHardlinks // encryption: 'encryptionPassword' and 'encryptedFilenames' is converted into an 'encryption' object using hush.js. Password is lost forever after conversion. const BACKUP_TARGET_FIELDS = [ 'id', 'label', 'provider', 'configJson', 'limitsJson', 'retentionJson', 'schedule', 'encryptionJson', 'format', 'main', 'creationTime', 'ts' ].join(','); @@ -397,7 +397,7 @@ async function getMountStatus(target) { } else if (target.provider === 'mountpoint') { hostPath = target.config.mountPoint; } else if (target.provider === 'filesystem') { - hostPath = target.config.backupFolder; + hostPath = target.config.backupDir; } else { return { state: 'active' }; } diff --git a/src/paths.js b/src/paths.js index 8e9a8190f..31ff00f84 100644 --- a/src/paths.js +++ b/src/paths.js @@ -28,7 +28,7 @@ exports = module.exports = { DEFAULT_BACKUP_DIR: '/var/backups', VOLUMES_MOUNT_DIR: '/mnt/volumes', - MANAGED_BACKUP_MOUNT_DIR: '/mnt/backups', + MANAGED_BACKUP_MOUNT_DIR: '/mnt/managedbackups', DOCKER_SOCKET_PATH: '/var/run/docker.sock', PLATFORM_DATA_DIR: path.join(baseDir(), 'platformdata'), diff --git a/src/routes/test/backuptargets-test.js b/src/routes/test/backuptargets-test.js index 006556db9..cf51aaba8 100644 --- a/src/routes/test/backuptargets-test.js +++ b/src/routes/test/backuptargets-test.js @@ -16,7 +16,7 @@ describe('Backups API', function () { const newTarget = { provider: 'filesystem', label: 'NewTarget', - config: { backupFolder: '/tmp/boxtest-newtarget' }, + config: { backupDir: '/tmp/boxtest-newtarget' }, format: 'tgz', retention: { keepWithinSecs: 60 * 60 }, schedule: '00 01 * * * *' @@ -25,7 +25,7 @@ describe('Backups API', function () { const encryptedTarget = { provider: 'filesystem', label: 'EncryptedTarget', - config: { backupFolder: '/tmp/boxtest-enctarget' }, + config: { backupDir: '/tmp/boxtest-enctarget' }, format: 'rsync', retention: { keepMonthly: 60 }, schedule: '* 1 * * * *', @@ -267,7 +267,7 @@ describe('Backups API', function () { describe('config', function () { const someConfig = { - backupFolder: '/tmp/boxtest-someconfig', + backupDir: '/tmp/boxtest-someconfig', }; it('cannot set invalid config', async function () { @@ -287,7 +287,7 @@ describe('Backups API', function () { expect(response.status).to.equal(200); const result = await backupTargets.get(newTarget.id); - expect(result.config.backupFolder).to.be(someConfig.backupFolder); + expect(result.config.backupDir).to.be(someConfig.backupDir); }); }); diff --git a/src/routes/test/common.js b/src/routes/test/common.js index df90014c1..193db43c0 100644 --- a/src/routes/test/common.js +++ b/src/routes/test/common.js @@ -121,7 +121,7 @@ async function setupServer() { const id = await backupTargets.add({ provider: 'filesystem', label: 'Default', - config: { backupFolder: '/tmp/boxtest' }, + config: { backupDir: '/tmp/boxtest' }, format: 'tgz', retention: { keepWithinSecs: 2 * 24 * 60 * 60 }, schedule: '00 00 23 * * *' diff --git a/src/storage/filesystem.js b/src/storage/filesystem.js index fdda7856e..faf8b982b 100644 --- a/src/storage/filesystem.js +++ b/src/storage/filesystem.js @@ -47,7 +47,7 @@ function getRootPath(config) { } else if (config._provider === mounts.MOUNT_TYPE_MOUNTPOINT) { return path.join(config.mountPoint, prefix); } else if (config._provider === mounts.MOUNT_TYPE_FILESYSTEM) { - return path.join(config.backupFolder, prefix); + return path.join(config.backupDir, prefix); } throw new BoxError(BoxError.INTERNAL_ERROR, `Unhandled provider: ${config._provider}`); @@ -239,15 +239,15 @@ async function removeDir(config, remotePathPrefix, progressCallback) { function validateDestDir(dir) { assert.strictEqual(typeof dir, 'string'); - if (path.normalize(dir) !== dir) return new BoxError(BoxError.BAD_FIELD, 'backupFolder/mountpoint must contain a normalized path'); - if (!path.isAbsolute(dir)) return new BoxError(BoxError.BAD_FIELD, 'backupFolder/mountpoint must be an absolute path'); + if (path.normalize(dir) !== dir) return new BoxError(BoxError.BAD_FIELD, 'backupDir/mountpoint must contain a normalized path'); + if (!path.isAbsolute(dir)) return new BoxError(BoxError.BAD_FIELD, 'backupDir/mountpoint must be an absolute path'); - if (dir === '/') return new BoxError(BoxError.BAD_FIELD, 'backupFolder/mountpoint cannot be /'); + if (dir === '/') return new BoxError(BoxError.BAD_FIELD, 'backupDir/mountpoint cannot be /'); if (!dir.endsWith('/')) dir = dir + '/'; // ensure trailing slash for the prefix matching to work const PROTECTED_PREFIXES = [ '/boot/', '/usr/', '/bin/', '/lib/', '/root/', '/var/lib/', paths.baseDir() ]; - if (PROTECTED_PREFIXES.some(p => dir.startsWith(p))) return new BoxError(BoxError.BAD_FIELD, 'backupFolder path is protected'); + if (PROTECTED_PREFIXES.some(p => dir.startsWith(p))) return new BoxError(BoxError.BAD_FIELD, 'backupDir path is protected'); return null; } @@ -318,8 +318,8 @@ async function verifyConfig({ id, provider, config }) { } if (provider === mounts.MOUNT_TYPE_FILESYSTEM) { - if (!config.backupFolder || typeof config.backupFolder !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'backupFolder must be non-empty string'); - const error = validateDestDir(config.backupFolder); + if (!config.backupDir || typeof config.backupDir !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'backupDir must be non-empty string'); + const error = validateDestDir(config.backupDir); if (error) throw error; } else { if (mounts.isManagedProvider(provider)) { @@ -341,7 +341,7 @@ async function verifyConfig({ id, provider, config }) { } } - const tmp = _.pick(config, ['noHardlinks', 'chown', 'preserveAttributes', 'backupFolder', 'prefix', 'mountOptions', 'mountPoint']); + const tmp = _.pick(config, ['noHardlinks', 'chown', 'preserveAttributes', 'backupDir', 'prefix', 'mountOptions', 'mountPoint']); const newConfig = { _provider: provider, _managedMountPath: managedMountPath, ...tmp }; const fullPath = getRootPath(newConfig); diff --git a/src/system.js b/src/system.js index 193bb2b3d..5a4149546 100644 --- a/src/system.js +++ b/src/system.js @@ -127,13 +127,13 @@ async function getFilesystems() { for (const backupTarget of await backupTargets.list(1, 100)) { if (backupTarget.provider === 'filesystem') { - const [, dfResult] = await safe(df.file(backupTarget.config.backupFolder)); + const [, dfResult] = await safe(df.file(backupTarget.config.backupDir)); const filesystem = dfResult?.filesystem || rootDisk.filesystem; - if (filesystems[filesystem]) filesystems[filesystem].contents.push({ type: 'cloudron-backup', id: backupTarget.id, path: backupTarget.config.backupFolder }); + if (filesystems[filesystem]) filesystems[filesystem].contents.push({ type: 'cloudron-backup', id: backupTarget.id, path: backupTarget.config.backupDir }); } // often the default backup dir is not cleaned up - if (backupTarget.provider !== 'filesystem' || backupTarget.config.backupFolder !== paths.DEFAULT_BACKUP_DIR) { + if (backupTarget.provider !== 'filesystem' || backupTarget.config.backupDir !== paths.DEFAULT_BACKUP_DIR) { const [, dfResult] = await safe(df.file(paths.DEFAULT_BACKUP_DIR)); const filesystem = dfResult?.filesystem || rootDisk.filesystem; if (filesystems[filesystem]) filesystems[filesystem].contents.push({ type: 'cloudron-backup-default', id: 'cloudron-backup-default', path: paths.DEFAULT_BACKUP_DIR }); diff --git a/src/test/backupcleaner-test.js b/src/test/backupcleaner-test.js index 5aab8b986..4e9a2403a 100644 --- a/src/test/backupcleaner-test.js +++ b/src/test/backupcleaner-test.js @@ -237,7 +237,7 @@ describe('backup cleaner', function () { target = await getDefaultBackupTarget(); await backupTargets.setConfig(target, { provider: 'filesystem', - backupFolder: '/tmp/someplace', + backupDir: '/tmp/someplace', }, auditSource); await backupTargets.setRetention(target, { keepWithinSecs: 1 }, auditSource); await backupTargets.setSchedule(target, '00 00 23 * * *', auditSource); diff --git a/src/test/backuptargets-test.js b/src/test/backuptargets-test.js index d514c8a78..ecb8c6aec 100644 --- a/src/test/backuptargets-test.js +++ b/src/test/backuptargets-test.js @@ -32,7 +32,7 @@ describe('backups', function () { it('can get backup target', async function () { const backupTarget = await backupTargets.get(defaultBackupTarget.id); expect(backupTarget.provider).to.be('filesystem'); - expect(backupTarget.config.backupFolder).to.be.ok(); // the test sets this to some tmp location + expect(backupTarget.config.backupDir).to.be.ok(); // the test sets this to some tmp location expect(backupTarget.format).to.be('tgz'); expect(backupTarget.encryption).to.be(null); }); @@ -43,11 +43,11 @@ describe('backups', function () { }); it('can set backup config', async function () { - const newConfig = Object.assign({}, defaultBackupTarget.config, { backupFolder: '/tmp/backups' }); + const newConfig = Object.assign({}, defaultBackupTarget.config, { backupDir: '/tmp/backups' }); await backupTargets.setConfig(defaultBackupTarget, newConfig, auditSource); const result = await backupTargets.get(defaultBackupTarget.id); - expect(result.config.backupFolder).to.be('/tmp/backups'); + expect(result.config.backupDir).to.be('/tmp/backups'); }); it('cannot set invalid schedule', async function () { diff --git a/src/test/backuptask-test.js b/src/test/backuptask-test.js index 408d380d8..800b2562b 100644 --- a/src/test/backuptask-test.js +++ b/src/test/backuptask-test.js @@ -27,13 +27,13 @@ describe('backuptask', function () { const backupConfig = { provider: 'filesystem', - backupFolder: path.join(os.tmpdir(), 'backupstask-test-filesystem'), + backupDir: path.join(os.tmpdir(), 'backupstask-test-filesystem'), }; let defaultBackupTarget; before(async function () { - fs.rmSync(backupConfig.backupFolder, { recursive: true, force: true }); + fs.rmSync(backupConfig.backupDir, { recursive: true, force: true }); defaultBackupTarget = await getDefaultBackupTarget(); await backupTargets.setConfig(defaultBackupTarget, backupConfig, auditSource); }); @@ -69,8 +69,8 @@ describe('backuptask', function () { } const result = await createBackup(defaultBackupTarget); - expect(fs.statSync(path.join(backupConfig.backupFolder, 'snapshot/box.tar.gz')).nlink).to.be(2); // hard linked to a rotated backup - expect(fs.statSync(path.join(backupConfig.backupFolder, result.remotePath)).nlink).to.be(2); + expect(fs.statSync(path.join(backupConfig.backupDir, 'snapshot/box.tar.gz')).nlink).to.be(2); // hard linked to a rotated backup + expect(fs.statSync(path.join(backupConfig.backupDir, result.remotePath)).nlink).to.be(2); backupInfo1 = result; }); @@ -83,13 +83,13 @@ describe('backuptask', function () { } const result = await createBackup(defaultBackupTarget); - expect(fs.statSync(path.join(backupConfig.backupFolder, 'snapshot/box.tar.gz')).nlink).to.be(2); // hard linked to a rotated backup - expect(fs.statSync(path.join(backupConfig.backupFolder, result.remotePath)).nlink).to.be(2); // hard linked to new backup - expect(fs.statSync(path.join(backupConfig.backupFolder, backupInfo1.remotePath)).nlink).to.be(1); // not hard linked anymore + expect(fs.statSync(path.join(backupConfig.backupDir, 'snapshot/box.tar.gz')).nlink).to.be(2); // hard linked to a rotated backup + expect(fs.statSync(path.join(backupConfig.backupDir, result.remotePath)).nlink).to.be(2); // hard linked to new backup + expect(fs.statSync(path.join(backupConfig.backupDir, backupInfo1.remotePath)).nlink).to.be(1); // not hard linked anymore }); it('cleanup', function () { - fs.rmSync(backupConfig.backupFolder, { recursive: true, force: true }); + fs.rmSync(backupConfig.backupDir, { recursive: true, force: true }); }); }); }); diff --git a/src/test/common.js b/src/test/common.js index 3b8da53b1..0935eaaeb 100644 --- a/src/test/common.js +++ b/src/test/common.js @@ -225,7 +225,7 @@ async function databaseSetup() { const id = await backupTargets.add({ provider: 'filesystem', label: 'Default', - config: { backupFolder: '/tmp/boxtest' }, + config: { backupDir: '/tmp/boxtest' }, format: 'tgz', retention: { keepWithinSecs: 2 * 24 * 60 * 60 }, schedule: '00 00 23 * * *' diff --git a/src/test/storage-provider-test.js b/src/test/storage-provider-test.js index 7484b6e84..56318d67d 100644 --- a/src/test/storage-provider-test.js +++ b/src/test/storage-provider-test.js @@ -33,7 +33,7 @@ describe('Storage', function () { const gBackupConfig = { key: 'key', - backupFolder: null, + backupDir: null, prefix: 'someprefix' }; @@ -42,7 +42,7 @@ describe('Storage', function () { before(async function () { gTmpFolder = fs.mkdtempSync(path.join(os.tmpdir(), 'filesystem-storage-test_')); defaultBackupTarget = await getDefaultBackupTarget(); - gBackupConfig.backupFolder = path.join(gTmpFolder, 'backups/'); + gBackupConfig.backupDir = path.join(gTmpFolder, 'backups/'); }); after(function (done) { @@ -51,20 +51,20 @@ describe('Storage', function () { }); it('fails to set backup storage for bad folder', async function () { - const tmp = Object.assign({}, gBackupConfig, { backupFolder: '/root/oof' }); + const tmp = Object.assign({}, gBackupConfig, { backupDir: '/root/oof' }); const [error] = await safe(backupTargets.setConfig(defaultBackupTarget, tmp, auditSource)); expect(error.reason).to.equal(BoxError.BAD_FIELD); }); it('succeeds to set backup storage', async function () { await backupTargets.setConfig(defaultBackupTarget, gBackupConfig, auditSource); - expect(fs.existsSync(path.join(gBackupConfig.backupFolder, 'someprefix/snapshot'))).to.be(true); // auto-created + expect(fs.existsSync(path.join(gBackupConfig.backupDir, 'someprefix/snapshot'))).to.be(true); // auto-created }); it('can upload', async function () { const sourceFile = path.join(__dirname, 'storage/data/test.txt'); const sourceStream = fs.createReadStream(sourceFile); - const destFile = path.join(gBackupConfig.backupFolder, gBackupConfig.prefix, '/uploadtest/test.txt'); + const destFile = path.join(gBackupConfig.backupDir, gBackupConfig.prefix, '/uploadtest/test.txt'); const uploader = await filesystem.upload(gBackupConfig, 'uploadtest/test.txt'); await stream.pipeline(sourceStream, uploader.stream); await uploader.finish(); @@ -75,7 +75,7 @@ describe('Storage', function () { xit('upload waits for empty file to be created', async function () { const sourceFile = path.join(__dirname, 'storage/data/empty'); const sourceStream = fs.createReadStream(sourceFile); - const destFile = path.join(gBackupConfig.backupFolder, gBackupConfig.prefix, '/uploadtest/empty'); + const destFile = path.join(gBackupConfig.backupDir, gBackupConfig.prefix, '/uploadtest/empty'); const uploader = await filesystem.upload(gBackupConfig, destFile); await stream.pipeline(sourceStream, uploader.stream); await uploader.finish(); @@ -86,7 +86,7 @@ describe('Storage', function () { it('upload unlinks old file', async function () { const sourceFile = path.join(__dirname, 'storage/data/test.txt'); const sourceStream = fs.createReadStream(sourceFile); - const destFile = path.join(gBackupConfig.backupFolder, gBackupConfig.prefix, '/uploadtest/test.txt'); + const destFile = path.join(gBackupConfig.backupDir, gBackupConfig.prefix, '/uploadtest/test.txt'); const oldStat = fs.statSync(destFile); const uploader = await filesystem.upload(gBackupConfig, 'uploadtest/test.txt'); await stream.pipeline(sourceStream, uploader.stream); @@ -97,7 +97,7 @@ describe('Storage', function () { }); it('can download file', async function () { - const sourceFile = path.join(gBackupConfig.backupFolder, gBackupConfig.prefix, '/uploadtest/test.txt'); + const sourceFile = path.join(gBackupConfig.backupDir, gBackupConfig.prefix, '/uploadtest/test.txt'); const [error, stream] = await safe(filesystem.download(gBackupConfig, 'uploadtest/test.txt')); expect(error).to.be(null); expect(stream).to.be.an('object'); @@ -112,7 +112,7 @@ describe('Storage', function () { it('list dir lists the source dir', async function () { const sourceDir = path.join(__dirname, 'storage'); - execSync(`cp -r ${sourceDir} ${gBackupConfig.backupFolder}/${gBackupConfig.prefix}`, { encoding: 'utf8' }); + execSync(`cp -r ${sourceDir} ${gBackupConfig.backupDir}/${gBackupConfig.prefix}`, { encoding: 'utf8' }); let allFiles = [], marker = null; while (true) { @@ -127,22 +127,22 @@ describe('Storage', function () { }); it('can copy', async function () { - // const sourceFile = path.join(gBackupConfig.backupFolder, gBackupConfig.prefix, '/uploadtest/test.txt'); // keep the test within same device - const destFile = path.join(gBackupConfig.backupFolder, gBackupConfig.prefix, '/uploadtest/test-hardlink.txt'); + // const sourceFile = path.join(gBackupConfig.backupDir, gBackupConfig.prefix, '/uploadtest/test.txt'); // keep the test within same device + const destFile = path.join(gBackupConfig.backupDir, gBackupConfig.prefix, '/uploadtest/test-hardlink.txt'); await filesystem.copy(gBackupConfig, 'uploadtest/test.txt', 'uploadtest/test-hardlink.txt', () => {}); expect(fs.statSync(destFile).nlink).to.be(2); // created a hardlink }); it('can remove file', async function () { - const sourceFile = path.join(gBackupConfig.backupFolder, gBackupConfig.prefix, '/uploadtest/test-hardlink.txt'); + const sourceFile = path.join(gBackupConfig.backupDir, gBackupConfig.prefix, '/uploadtest/test-hardlink.txt'); await filesystem.remove(gBackupConfig, 'uploadtest/test-hardlink.txt'); expect(fs.existsSync(sourceFile)).to.be(false); }); it('can remove empty dir', async function () { - const sourceDir = path.join(gBackupConfig.backupFolder, gBackupConfig.prefix, 'emptydir'); + const sourceDir = path.join(gBackupConfig.backupDir, gBackupConfig.prefix, 'emptydir'); fs.mkdirSync(sourceDir); await filesystem.remove(gBackupConfig, 'emptydir', () => {});