jjkiers/cloudron-box
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

storage: pass limits object to backend

Browse Source
This commit is contained in:
Girish Ramakrishnan
2025-11-14 13:18:21 +01:00
parent cb30a57a59
commit e106dcd76a
8 changed files with 37 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
src/storage/filesystem.js
View File

@@ -103,8 +103,9 @@ function hasChownSupportSync(config) {
}
}
async function upload(config, remotePath) {
async function upload(config, limits, remotePath) {
assert.strictEqual(typeof config, 'object');
assert.strictEqual(typeof limits, 'object');
assert.strictEqual(typeof remotePath, 'string');
const fullRemotePath = path.join(getRootPath(config), remotePath);
@@ -234,8 +235,9 @@ async function copy(config, fromPath, toPath, progressCallback) {
return await copyInternal(config, fromPath, toPath, { recursive: false }, progressCallback);
}
async function copyDir(config, fromPath, toPath, progressCallback) {
async function copyDir(config, limits, fromPath, toPath, progressCallback) {
assert.strictEqual(typeof config, 'object');
assert.strictEqual(typeof limits, 'object');
assert.strictEqual(typeof fromPath, 'string');
assert.strictEqual(typeof toPath, 'string');
assert.strictEqual(typeof progressCallback, 'function');
@@ -258,8 +260,9 @@ async function remove(config, remotePath) {
}
}
async function removeDir(config, remotePathPrefix, progressCallback) {
async function removeDir(config, limits, remotePathPrefix, progressCallback) {
assert.strictEqual(typeof config, 'object');
assert.strictEqual(typeof limits, 'object');
assert.strictEqual(typeof remotePathPrefix, 'string');
assert.strictEqual(typeof progressCallback, 'function');

14
src/storage/gcs.js
View File

@@ -67,8 +67,9 @@ async function getStatus(config) {
return { state: 'active', message: '' };
}
async function upload(config, remotePath) {
async function upload(config, limits, remotePath) {
assert.strictEqual(typeof config, 'object');
assert.strictEqual(typeof limits, 'object');
assert.strictEqual(typeof remotePath, 'string');
const fullRemotePath = path.join(config.prefix, remotePath);
@@ -156,14 +157,15 @@ async function copy(config, fullFromPath, fullToPath, progressCallback) {
if (copyError) throw new BoxError(BoxError.EXTERNAL_ERROR, copyError.message);
}
async function copyDir(config, fromPath, toPath, progressCallback) {
async function copyDir(config, limits, fromPath, toPath, progressCallback) {
assert.strictEqual(typeof config, 'object');
assert.strictEqual(typeof limits, 'object');
assert.strictEqual(typeof fromPath, 'string');
assert.strictEqual(typeof toPath, 'string');
assert.strictEqual(typeof progressCallback, 'function');
const batchSize = 1000;
const concurrency = config.limits?.copyConcurrency || 10;
const concurrency = limits.copyConcurrency || 10;
let total = 0;
let marker = null;
@@ -192,12 +194,14 @@ async function remove(config, remotePath) {
if (error) debug('removeBackups: Unable to remove %s (%s). Not fatal.', fullRemotePath, error.message);
}
async function removeDir(config, remotePathPrefix, progressCallback) {
async function removeDir(config, limits, remotePathPrefix, progressCallback) {
assert.strictEqual(typeof config, 'object');
assert.strictEqual(typeof limits, 'object');
assert.strictEqual(typeof remotePathPrefix, 'string');
assert.strictEqual(typeof progressCallback, 'function');
const batchSize = 1000, concurrency = config.limits?.deleteConcurrency || 10; // https://googleapis.dev/nodejs/storage/latest/Bucket.html#deleteFiles
const batchSize = 1000;
const concurrency = limits.deleteConcurrency || 10; // https://googleapis.dev/nodejs/storage/latest/Bucket.html#deleteFiles
let total = 0;
let marker = null;

9
src/storage/interface.js
View File

@@ -62,8 +62,9 @@ async function getStatus(config) {
throw new BoxError(BoxError.NOT_IMPLEMENTED, 'getStatus is not implemented');
}
async function upload(config, backupFilePath) {
async function upload(config, limits, backupFilePath) {
assert.strictEqual(typeof config, 'object');
assert.strictEqual(typeof limits, 'object');
assert.strictEqual(typeof backupFilePath, 'string');
// Result: { stream, finish() callback }
@@ -95,8 +96,9 @@ async function copy(config, oldFilePath, newFilePath, progressCallback) {
throw new BoxError(BoxError.NOT_IMPLEMENTED, 'copy is not implemented');
}
async function copyDir(config, oldFilePath, newFilePath, progressCallback) {
async function copyDir(config, limits, oldFilePath, newFilePath, progressCallback) {
assert.strictEqual(typeof config, 'object');
assert.strictEqual(typeof limits, 'object');
assert.strictEqual(typeof oldFilePath, 'string');
assert.strictEqual(typeof newFilePath, 'string');
assert.strictEqual(typeof progressCallback, 'function');
@@ -123,8 +125,9 @@ async function remove(config, filename) {
throw new BoxError(BoxError.NOT_IMPLEMENTED, 'remove is not implemented');
}
async function removeDir(config, pathPrefix, progressCallback) {
async function removeDir(config, limits, pathPrefix, progressCallback) {
assert.strictEqual(typeof config, 'object');
assert.strictEqual(typeof limits, 'object');
assert.strictEqual(typeof pathPrefix, 'string');
assert.strictEqual(typeof progressCallback, 'function');

15
src/storage/s3.js
View File

@@ -169,8 +169,9 @@ async function getStatus(config) {
return { state: 'active', message: '' };
}
async function upload(config, remotePath) {
async function upload(config, limits, remotePath) {
assert.strictEqual(typeof config, 'object');
assert.strictEqual(typeof limits, 'object');
assert.strictEqual(typeof remotePath, 'string');
const s3 = createS3Client(config, { retryStrategy: RETRY_STRATEGY });
@@ -179,7 +180,7 @@ async function upload(config, remotePath) {
// uploader will buffer at most queueSize * partSize bytes into memory at any given time.
// scaleway only supports 1000 parts per object (https://www.scaleway.com/en/docs/s3-multipart-upload/)
// s3: https://docs.aws.amazon.com/AmazonS3/latest/dev/qfacts.html (max 10k parts and no size limit on the last part!)
const partSize = config.limits?.uploadPartSize || (config._provider === 'scaleway-objectstorage' ? 100 * 1024 * 1024 : 10 * 1024 * 1024);
const partSize = limits.uploadPartSize || (config._provider === 'scaleway-objectstorage' ? 100 * 1024 * 1024 : 10 * 1024 * 1024);
const passThrough = new PassThrough();
@@ -487,14 +488,15 @@ async function copy(config, fromPath, toPath, progressCallback) {
return await copyInternal(config, fullFromPath, fullToPath, data.ContentLength, progressCallback);
}
async function copyDir(config, fromPath, toPath, progressCallback) {
async function copyDir(config, limits, fromPath, toPath, progressCallback) {
assert.strictEqual(typeof config, 'object');
assert.strictEqual(typeof limits, 'object');
assert.strictEqual(typeof fromPath, 'string');
assert.strictEqual(typeof toPath, 'string');
assert.strictEqual(typeof progressCallback, 'function');
let total = 0;
const concurrency = config.limits?.copyConcurrency || (config._provider === 's3' ? 500 : 10);
const concurrency = limits.copyConcurrency || (config._provider === 's3' ? 500 : 10);
progressCallback({ message: `Copying ${fromPath} to ${toPath} with concurrency of ${concurrency}` });
let marker = null;
@@ -548,8 +550,9 @@ function chunk(array, size) {
return result;
}
async function removeDir(config, remotePathPrefix, progressCallback) {
async function removeDir(config, limits, remotePathPrefix, progressCallback) {
assert.strictEqual(typeof config, 'object');
assert.strictEqual(typeof limits, 'object');
assert.strictEqual(typeof remotePathPrefix, 'string');
assert.strictEqual(typeof progressCallback, 'function');
@@ -564,7 +567,7 @@ async function removeDir(config, remotePathPrefix, progressCallback) {
const entries = batch.entries;
total += entries.length;
const chunkSize = config.limits?.deleteConcurrency || (config._provider !== 'digitalocean-spaces' ? 1000 : 100); // throttle objects in each request
const chunkSize = limits.deleteConcurrency || (config._provider !== 'digitalocean-spaces' ? 1000 : 100); // throttle objects in each request
const chunks = chunk(entries, chunkSize);
await async.eachSeries(chunks, async function deleteFiles(objects) {