From 4fba216af9eff0687b6cd55586340ffd910aeb0c Mon Sep 17 00:00:00 2001 From: Girish Ramakrishnan Date: Mon, 30 Sep 2019 20:42:37 -0700 Subject: [PATCH] scaleway: try to keep part numbers low --- src/storage/s3.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/storage/s3.js b/src/storage/s3.js index 039f63eea..5e938e881 100644 --- a/src/storage/s3.js +++ b/src/storage/s3.js @@ -95,7 +95,10 @@ function upload(apiConfig, backupFilePath, sourceStream, callback) { // s3.upload automatically does a multi-part upload. we set queueSize to 1 to reduce memory usage // uploader will buffer at most queueSize * partSize bytes into memory at any given time. - s3.upload(params, { partSize: 10 * 1024 * 1024, queueSize: 1 }, function (error, data) { + // scaleway only supports 1000 parts per object (https://www.scaleway.com/en/docs/s3-multipart-upload/) + const partSize = apiConfig.provider === 'scaleway-objectstorage' ? 100 * 1024 * 1024 : 10 * 1024 * 1024; + + s3.upload(params, { partSize, queueSize: 1 }, function (error, data) { if (error) { debug('Error uploading [%s]: s3 upload error.', backupFilePath, error); return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, `Error uploading ${backupFilePath}. Message: ${error.message} HTTP Code: ${error.code}`));