jjkiers/cloudron-box
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

backups: root ~~canal~~ path surgery

Browse Source 
remove rootPath and getBackupFilePath from the backup target and
make this backend specific.
This commit is contained in:
Girish Ramakrishnan
2025-08-02 01:46:29 +02:00
parent a01e1bad0f
commit c935744f4c
15 changed files with 378 additions and 373 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/test/storage-provider-test.js
+86 -96
View File
@@ -9,6 +9,7 @@
const backupTargets = require('../backuptargets.js'),
BoxError = require('../boxerror.js'),
common = require('./common.js'),
consumers = require('node:stream/consumers'),
execSync = require('child_process').execSync,
expect = require('expect.js'),
filesystem = require('../storage/filesystem.js'),
@@ -21,8 +22,6 @@ const backupTargets = require('../backuptargets.js'),
safe = require('safetydance'),
stream = require('stream/promises');
const chunk = s3._chunk;
describe('Storage', function () {
const { setup, cleanup, getDefaultBackupTarget, auditSource } = common;
@@ -35,6 +34,7 @@ describe('Storage', function () {
const gBackupConfig = {
key: 'key',
backupFolder: null,
prefix: 'someprefix'
};
let defaultBackupTarget;
@@ -58,24 +58,24 @@ describe('Storage', function () {
it('succeeds to set backup storage', async function () {
await backupTargets.setConfig(defaultBackupTarget, gBackupConfig, auditSource);
expect(fs.existsSync(path.join(gBackupConfig.backupFolder, 'snapshot'))).to.be(true); // auto-created
expect(fs.existsSync(path.join(gBackupConfig.backupFolder, 'someprefix/snapshot'))).to.be(true); // auto-created
});
it('can upload', async function () {
const sourceFile = path.join(__dirname, 'storage/data/test.txt');
const sourceStream = fs.createReadStream(sourceFile);
const destFile = gTmpFolder + '/uploadtest/test.txt';
const uploader = await filesystem.upload(gBackupConfig, destFile);
const destFile = path.join(gBackupConfig.backupFolder, gBackupConfig.prefix, '/uploadtest/test.txt');
const uploader = await filesystem.upload(gBackupConfig, 'uploadtest/test.txt');
await stream.pipeline(sourceStream, uploader.stream);
await uploader.finish();
expect(fs.existsSync(destFile));
expect(fs.statSync(sourceFile).size).to.be(fs.statSync(destFile).size);
});
it('upload waits for empty file to be created', async function () {
xit('upload waits for empty file to be created', async function () {
const sourceFile = path.join(__dirname, 'storage/data/empty');
const sourceStream = fs.createReadStream(sourceFile);
const destFile = gTmpFolder + '/uploadtest/empty';
const destFile = path.join(gBackupConfig.backupFolder, gBackupConfig.prefix, '/uploadtest/empty');
const uploader = await filesystem.upload(gBackupConfig, destFile);
await stream.pipeline(sourceStream, uploader.stream);
await uploader.finish();
@@ -86,9 +86,9 @@ describe('Storage', function () {
it('upload unlinks old file', async function () {
const sourceFile = path.join(__dirname, 'storage/data/test.txt');
const sourceStream = fs.createReadStream(sourceFile);
const destFile = gTmpFolder + '/uploadtest/test.txt';
const destFile = path.join(gBackupConfig.backupFolder, gBackupConfig.prefix, '/uploadtest/test.txt');
const oldStat = fs.statSync(destFile);
const uploader = await filesystem.upload(gBackupConfig, destFile);
const uploader = await filesystem.upload(gBackupConfig, 'uploadtest/test.txt');
await stream.pipeline(sourceStream, uploader.stream);
await uploader.finish();
expect(fs.existsSync(destFile)).to.be(true);
@@ -97,55 +97,55 @@ describe('Storage', function () {
});
it('can download file', async function () {
const sourceFile = gTmpFolder + '/uploadtest/test.txt';
const [error, stream] = await safe(filesystem.download(gBackupConfig, sourceFile));
const sourceFile = path.join(gBackupConfig.backupFolder, gBackupConfig.prefix, '/uploadtest/test.txt');
const [error, stream] = await safe(filesystem.download(gBackupConfig, 'uploadtest/test.txt'));
expect(error).to.be(null);
expect(stream).to.be.an('object');
const data = await consumers.buffer(stream);
expect(fs.readFileSync(sourceFile)).to.eql(data); // buffer compare
});
it('download errors for missing file', async function () {
const sourceFile = gTmpFolder + '/uploadtest/missing';
const [error] = await safe(filesystem.download(gBackupConfig, sourceFile));
const [error] = await safe(filesystem.download(gBackupConfig, 'uploadtest/missing'));
expect(error.reason).to.be(BoxError.NOT_FOUND);
});
it('list dir lists the source dir', async function () {
const sourceDir = path.join(__dirname, 'storage');
execSync(`cp -r ${sourceDir} ${gBackupConfig.backupFolder}/${gBackupConfig.prefix}`, { encoding: 'utf8' });
let allFiles = [], marker = null;
while (true) {
const result = await filesystem.listDir(gBackupConfig, sourceDir, 1, marker);
const result = await filesystem.listDir(gBackupConfig, 'storage', 1, marker);
allFiles = allFiles.concat(result.entries);
if (!result.marker) break;
marker = result.marker;
}
const expectedFiles = execSync(`find ${sourceDir} -type f`, { encoding: 'utf8' }).trim().split('\n');
const expectedFiles = execSync(`find . -type f -printf '%P\n'`, { cwd: sourceDir, encoding: 'utf8' }).trim().split('\n');
expect(allFiles.map(function (f) { return f.fullPath; }).sort()).to.eql(expectedFiles.sort());
});
it('can copy', async function () {
const sourceFile = gTmpFolder + '/uploadtest/test.txt'; // keep the test within save device
const destFile = gTmpFolder + '/uploadtest/test-hardlink.txt';
// const sourceFile = path.join(gBackupConfig.backupFolder, gBackupConfig.prefix, '/uploadtest/test.txt'); // keep the test within same device
const destFile = path.join(gBackupConfig.backupFolder, gBackupConfig.prefix, '/uploadtest/test-hardlink.txt');
await filesystem.copy(gBackupConfig, sourceFile, destFile, () => {});
await filesystem.copy(gBackupConfig, 'uploadtest/test.txt', 'uploadtest/test-hardlink.txt', () => {});
expect(fs.statSync(destFile).nlink).to.be(2); // created a hardlink
});
it('can remove file', async function () {
const sourceFile = gTmpFolder + '/uploadtest/test-hardlink.txt';
const sourceFile = path.join(gBackupConfig.backupFolder, gBackupConfig.prefix, '/uploadtest/test-hardlink.txt');
await filesystem.remove(gBackupConfig, sourceFile);
await filesystem.remove(gBackupConfig, 'uploadtest/test-hardlink.txt');
expect(fs.existsSync(sourceFile)).to.be(false);
});
it('can remove empty dir', async function () {
const sourceDir = gTmpFolder + '/emptydir';
const sourceDir = path.join(gBackupConfig.backupFolder, gBackupConfig.prefix, 'emptydir');
fs.mkdirSync(sourceDir);
await filesystem.remove(gBackupConfig, sourceDir, () => {});
await filesystem.remove(gBackupConfig, 'emptydir', () => {});
expect(fs.existsSync(sourceDir)).to.be(false);
});
});
@@ -196,15 +196,15 @@ describe('Storage', function () {
region: 'eu-central-1',
format: 'tgz'
};
const bucketPath = path.join(basePath, backupConfig.bucket);
const bucketPath = path.join(basePath, backupConfig.bucket, backupConfig.prefix);
const bucketPathNoPrefix = path.join(basePath, backupConfig.bucket);
class S3MockUpload {
constructor(args) { // { client: s3, params, partSize, queueSize: 3, leavePartsOnError: false }
console.log(basePath, args.params.Bucket, args.params.Key);
// console.log('S3MockUpload constructor:', basePath, args.params.Bucket, args.params.Key);
const destFilePath = path.join(basePath, args.params.Bucket, args.params.Key);
fs.mkdirSync(path.dirname(destFilePath), { recursive: true });
this.pipeline = stream.pipeline(args.params.Body, fs.createWriteStream(destFilePath));
console.log(destFilePath);
}
on() {}
@@ -227,29 +227,31 @@ describe('Storage', function () {
expect(params.Bucket).to.be(backupConfig.bucket);
return {
Contents: [{
Key: 'uploadtest/test.txt',
Key: `${backupConfig.prefix}/uploadtest/test.txt`,
Size: 23
}, {
Key: 'uploadtest/C++.gitignore',
Key: `${backupConfig.prefix}/uploadtest/C++.gitignore`,
Size: 23
}]
};
}
async copyObject(params) {
console.log(path.join(basePath, params.CopySource), path.join(bucketPath, params.Key));
await fs.promises.mkdir(path.dirname(path.join(bucketPath, params.Key)), { recursive: true });
await fs.promises.copyFile(path.join(basePath, params.CopySource.replace(/%2B/g, '+')), path.join(bucketPath, params.Key)); // CopySource already has the bucket path!
// CopySource already has the bucket path!
// Key already has prefix but no bucket ptah!
// console.log('Copying:', path.join(basePath, params.CopySource), path.join(bucketPathNoPrefix, params.Key));
await fs.promises.mkdir(path.dirname(path.join(bucketPathNoPrefix, params.Key)), { recursive: true });
await fs.promises.copyFile(path.join(basePath, params.CopySource.replace(/%2B/g, '+')), path.join(bucketPathNoPrefix, params.Key));
}
async deleteObject(params) {
expect(params.Bucket).to.be(backupConfig.bucket);
fs.rmSync(path.join(bucketPath, params.Key));
fs.rmSync(path.join(bucketPathNoPrefix, params.Key));
}
async deleteObjects(params) {
expect(params.Bucket).to.be(backupConfig.bucket);
params.Delete.Objects.forEach(o => fs.rmSync(path.join(bucketPath, o.Key)));
params.Delete.Objects.forEach(o => fs.rmSync(path.join(bucketPathNoPrefix, o.Key)));
}
}
@@ -284,7 +286,7 @@ describe('Storage', function () {
});
it('list dir lists contents of source dir', async function () {
let allFiles = [ ], marker = null;
let allFiles = [], marker = null;
while (true) {
const result = await s3.listDir(backupConfig, '', 1, marker);
@@ -317,7 +319,7 @@ describe('Storage', function () {
});
describe('gcs', function () {
const gBackupConfig = {
const backupConfig = {
provider: 'gcs',
key: '',
prefix: 'unit.test',
@@ -329,70 +331,64 @@ describe('Storage', function () {
}
};
const GCSMockBasePath = path.join(os.tmpdir(), 'gcs-backup-test-buckets/');
const basePath = path.join(os.tmpdir(), 'gcs-backup-test-buckets/');
const bucketPath = path.join(basePath, backupConfig.bucket, backupConfig.prefix);
const bucketPathNoPrefix = path.join(basePath, backupConfig.bucket);
class GCSMockBucket {
constructor(name) {
expect(name).to.be(gBackupConfig.bucket);
expect(name).to.be(backupConfig.bucket);
}
file(filename) {
function ensurePathWritable(filename) {
filename = GCSMockBasePath + filename;
fs.mkdirSync(path.dirname(filename), { recursive: true });
return filename;
file(key) { // already has prefix
// console.log('gcs file object:', key);
function getFullWritablePath(key) {
const fullPath = path.join(bucketPathNoPrefix, key);
fs.mkdirSync(path.dirname(fullPath), { recursive: true });
console.log(fullPath);
return fullPath;
}
return {
name: filename,
name: key,
createReadStream: function() {
return fs.createReadStream(ensurePathWritable(filename))
return fs.createReadStream(getFullWritablePath(key))
.on('error', function(e){
console.log('error createReadStream: '+filename);
console.log('error createReadStream: '+key);
if (e.code == 'ENOENT') { e.code = 404; }
this.emit('error', e);
});
},
createWriteStream: function() {
return fs.createWriteStream(ensurePathWritable(filename));
return fs.createWriteStream(getFullWritablePath(key));
},
delete: async function() {
await fs.promises.unlink(ensurePathWritable(filename));
await fs.promises.unlink(getFullWritablePath(key));
},
copy: function(dst, cb) {
function notFoundHandler(e) {
if (e && e.code == 'ENOENT') { e.code = 404; return cb(e); }
cb();
}
return fs.createReadStream(ensurePathWritable(filename))
.on('end', cb)
.on('error', notFoundHandler)
.pipe(fs.createWriteStream(ensurePathWritable(dst)))
.on('end', cb)
.on('error', notFoundHandler);
copy: async function(destKey) {
// console.log('gcs copy:', key, destKey);
await fs.promises.mkdir(path.dirname(path.join(bucketPathNoPrefix, destKey)), { recursive: true });
await fs.promises.copyFile(path.join(bucketPathNoPrefix, key), path.join(bucketPathNoPrefix, destKey));
}
};
}
async getFiles(q) {
const target = path.join(GCSMockBasePath, q.prefix);
const files = execSync(`find ${target} -type f`, { encoding: 'utf8' }).trim().split('\n');
const pageToken = q.pageToken || 0;
expect(q.maxResults).to.be.a('number');
expect(q.prefix).to.be.a('string');
const chunkedFiles = chunk(files, q.maxResults);
if (q.pageToken >= chunkedFiles.length) return [[], null];
const files = [{
name: `${backupConfig.prefix}/uploadtest/test.txt`,
}, {
name: `${backupConfig.prefix}/uploadtest/C++.gitignore`,
}];
const gFiles = chunkedFiles[pageToken].map(f => {
return this.file(path.relative(GCSMockBasePath, f));
});
q.pageToken = pageToken + 1;
return [ gFiles, q.pageToken < chunkedFiles.length ? q : null ];
return [ files, null ];
}
};
class GCSMock {
constructor(config) {
expect(config.projectId).to.be(gBackupConfig.projectId);
expect(config.credentials.private_key).to.be(gBackupConfig.credentials.private_key);
expect(config.projectId).to.be(backupConfig.projectId);
expect(config.credentials.private_key).to.be(backupConfig.credentials.private_key);
}
bucket(name) {
@@ -405,22 +401,24 @@ describe('Storage', function () {
});
after(function () {
fs.rmSync(GCSMockBasePath, { recursive: true, force: true });
fs.rmSync(basePath, { recursive: true, force: true });
delete globalThis.GCSMock;
});
it('can backup', async function () {
it('can upload', async function () {
const sourceFile = path.join(__dirname, 'storage/data/test.txt');
const sourceStream = fs.createReadStream(sourceFile);
const destKey = 'uploadtest/test.txt';
const uploader = await gcs.upload(gBackupConfig, destKey);
const uploader = await gcs.upload(backupConfig, destKey);
await stream.pipeline(sourceStream, uploader.stream);
await uploader.finish();
expect(fs.existsSync(path.join(bucketPath, destKey))).to.be(true);
expect(fs.statSync(path.join(bucketPath, destKey)).size).to.be(fs.statSync(sourceFile).size);
});
it('can download file', async function () {
const sourceKey = 'uploadtest/test.txt';
const [error, stream] = await safe(gcs.download(gBackupConfig, sourceKey));
const [error, stream] = await safe(gcs.download(backupConfig, sourceKey));
expect(error).to.be(null);
expect(stream).to.be.an('object');
});
@@ -429,39 +427,31 @@ describe('Storage', function () {
let allFiles = [ ], marker = null;
while (true) {
const result = await gcs.listDir(gBackupConfig, '', 1, marker);
const result = await gcs.listDir(backupConfig, '', 1, marker);
allFiles = allFiles.concat(result.entries);
if (!result.marker) break;
marker = result.marker;
}
expect(allFiles.map(function (f) { return f.fullPath; }).sort()).to.eql([ 'uploadtest/test.txt' ]);
expect(allFiles.map(function (f) { return f.fullPath; })).to.contain('uploadtest/test.txt');
});
xit('can copy', function (done) {
fs.writeFileSync(path.join(GCSMockBasePath, 'uploadtest/C++.gitignore'), 'special', 'utf8');
it('can copy', async function () {
fs.writeFileSync(path.join(bucketPath, 'uploadtest/C++.gitignore'), 'special', 'utf8');
const sourceKey = 'uploadtest';
const events = gcs.copy(gBackupConfig, sourceKey, 'uploadtest-copy');
events.on('done', function (error) {
const sourceFile = path.join(__dirname, 'storage/data/test.txt');
expect(error).to.be(null);
expect(fs.statSync(path.join(GCSMockBasePath, 'uploadtest-copy/test.txt')).size).to.be(fs.statSync(sourceFile).size);
expect(fs.statSync(path.join(GCSMockBasePath, 'uploadtest-copy/C++.gitignore')).size).to.be(7);
done();
});
await gcs.copy(backupConfig, 'uploadtest', 'uploadtest-copy', () => {});
const sourceFile = path.join(__dirname, 'storage/data/test.txt');
expect(fs.statSync(path.join(bucketPath, 'uploadtest-copy/test.txt')).size).to.be(fs.statSync(sourceFile).size);
expect(fs.statSync(path.join(bucketPath, 'uploadtest-copy/C++.gitignore')).size).to.be(7);
});
it('can remove file', async function () {
await gcs.remove(gBackupConfig, 'uploadtest-copy/test.txt');
expect(fs.existsSync(path.join(GCSMockBasePath, 'uploadtest-copy/test.txt'))).to.be(false);
await gcs.remove(backupConfig, 'uploadtest-copy/test.txt');
expect(fs.existsSync(path.join(basePath, 'uploadtest-copy/test.txt'))).to.be(false);
});
it('can remove non-existent dir', async function () {
await gcs.remove(gBackupConfig, 'blah', () => {});
await gcs.remove(backupConfig, 'blah', () => {});
});
});
});