jjkiers/cloudron-box
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

storage: rework listDir api to be a generator (like) function

Browse Source
This commit is contained in:
Girish Ramakrishnan
2025-02-12 18:46:54 +01:00
parent da0dcf65b3
commit 9888aa8c08
9 changed files with 143 additions and 180 deletions
Download Patch File Download Diff File Expand all files Collapse all files

77
src/test/storage-test.js
View File

@@ -18,7 +18,6 @@ const backups = require('../backups.js'),
noop = require('../storage/noop.js'),
os = require('os'),
path = require('path'),
{ readdirpPromise } = require('readdirp'),
s3 = require('../storage/s3.js'),
safe = require('safetydance'),
stream = require('stream/promises');
@@ -115,19 +114,19 @@ describe('Storage', function () {
expect(error.reason).to.be(BoxError.NOT_FOUND);
});
it('list dir lists the source dir', function (done) {
it('list dir lists the source dir', async function () {
const sourceDir = path.join(__dirname, 'storage');
let allFiles = [ ];
filesystem.listDir(gBackupConfig, sourceDir, 1, function (files, iteratorCallback) {
allFiles = allFiles.concat(files);
iteratorCallback();
}, function () {
const expectedFiles = execSync(`find ${sourceDir} -type f`, { encoding: 'utf8' }).trim().split('\n');
expect(allFiles.map(function (f) { return f.fullPath; }).sort()).to.eql(expectedFiles.sort());
let allFiles = [], marker = null;
while (true) {
const result = await filesystem.listDir(gBackupConfig, sourceDir, 1, marker);
allFiles = allFiles.concat(result.entries);
if (!result.marker) break;
marker = result.marker;
}
done();
});
const expectedFiles = execSync(`find ${sourceDir} -type f`, { encoding: 'utf8' }).trim().split('\n');
expect(allFiles.map(function (f) { return f.fullPath; }).sort()).to.eql(expectedFiles.sort());
});
it('can copy', async function () {
@@ -169,10 +168,10 @@ describe('Storage', function () {
expect(error).to.be.an(Error);
});
it('list dir contents of source dir', function (done) {
noop.listDir(gBackupConfig, 'sourceDir', 1000, function (files, iteratorDone) {
iteratorDone();
}, done);
it('list dir contents of source dir', async function () {
const result = await noop.listDir(gBackupConfig, 'sourceDir', 1000, null /* marker */);
expect(result.marker).to.be(null);
expect(result.entries).to.eql([]);
});
it('can copy', async function () {
@@ -232,16 +231,17 @@ describe('Storage', function () {
expect(stream).to.be.an('object');
});
it('list dir lists contents of source dir', function (done) {
let allFiles = [ ];
s3.listDir(gBackupConfig, '', 1, function (files, iteratorCallback) {
allFiles = allFiles.concat(files);
iteratorCallback();
}, function () {
expect(allFiles.map(function (f) { return f.fullPath; }).sort()).to.eql([ 'uploadtest/test.txt' ]);
it('list dir lists contents of source dir', async function () {
let allFiles = [ ], marker = null;
done();
});
while (true) {
const result = await s3.listDir(gBackupConfig, '', 1, marker);
allFiles = allFiles.concat(result.entries);
if (!result.marker) break;
marker = result.marker;
}
expect(allFiles.map(function (f) { return f.fullPath; }).sort()).to.eql([ 'uploadtest/test.txt' ]);
});
it('can copy', async function () {
@@ -325,20 +325,20 @@ describe('Storage', function () {
return {
file,
getFiles: async function(q, cb) {
const target = GCSMockBasePath + q.prefix;
const files = (await readdirpPromise(target)).map(f => f.fullPath);
getFiles: async function(q) {
const target = path.join(GCSMockBasePath, q.prefix);
const files = execSync(`find ${target} -type f`, { encoding: 'utf8' }).trim().split('\n');
const pageToken = q.pageToken || 0;
const chunkedFiles = chunk(files, q.maxResults);
if (q.pageToken >= chunkedFiles.length) return cb(null, []);
if (q.pageToken >= chunkedFiles.length) return [[], null];
const gFiles = chunkedFiles[pageToken].map(function(f) {
return file(path.relative(GCSMockBasePath, f)); //convert to gcs
});
q.pageToken = pageToken + 1;
cb(null, gFiles, q.pageToken < chunkedFiles.length ? q : null);
return [ gFiles, q.pageToken < chunkedFiles.length ? q : null ];
}
};
}};
@@ -368,16 +368,17 @@ describe('Storage', function () {
expect(stream).to.be.an('object');
});
it('list dir lists contents of source dir', function (done) {
let allFiles = [ ];
gcs.listDir(gBackupConfig, '', 1, function (files, iteratorCallback) {
allFiles = allFiles.concat(files);
iteratorCallback();
}, function () {
expect(allFiles.map(function (f) { return f.fullPath; }).sort()).to.eql([ 'uploadtest/test.txt' ]);
it('list dir lists contents of source dir', async function () {
let allFiles = [ ], marker = null;
done();
});
while (true) {
const result = await gcs.listDir(gBackupConfig, '', 1, marker);
allFiles = allFiles.concat(result.entries);
if (!result.marker) break;
marker = result.marker;
}
expect(allFiles.map(function (f) { return f.fullPath; }).sort()).to.eql([ 'uploadtest/test.txt' ]);
});
xit('can copy', function (done) {