diff --git a/src/storage/s3.js b/src/storage/s3.js index 1868a43fd..4eab6b2c8 100644 --- a/src/storage/s3.js +++ b/src/storage/s3.js @@ -59,33 +59,47 @@ function createS3Client(apiConfig, options) { secretAccessKey: apiConfig.secretAccessKey }; - const requestHandler = new NodeHttpHandler({ - connectionTimeout: 60000, - socketTimeout: 20 * 60 * 1000 - }); + class CustomHttpHandler extends NodeHttpHandler { + // AWS decided to use CRC32 for checksums. The Client SDK has then been changed to set this for requests as default. https://github.com/aws/aws-sdk-js-v3/blob/main/supplemental-docs/MD5_FALLBACK.md + // requestChecksumCalculation: "WHEN_REQUIRED", responseChecksumValidation: "WHEN_REQUIRED", "checksumAlgorithm": "md5" all don't work + // Non-AWS doesn't support this. It seems when there is a request body, the sdk always sends Expect 101-continue header before sending request body. DO Spaces just doesn't respond to this. + // see also: https://github.com/aws/aws-sdk-js-v3/issues/6810 https://github.com/aws/aws-sdk-js-v3/issues/6819 https://github.com/aws/aws-sdk-js-v3/issues/6761 + #removeExpectHeader; + constructor(options) { + super(options); + this.#removeExpectHeader = options.removeExpectHeader; + + if (options.http) this.agent = new http.Agent({}); + if (options.acceptSelfSignedCerts) this.agent = new https.Agent({ rejectUnauthorized: false }); + } + + async handle(request, options) { + if (this.#removeExpectHeader && request.headers?.Expect) delete request.headers['Expect']; + return super.handle(request, options); + } + } + + // aws s3 endpoint names come from the SDK + const isHttps = apiConfig.endpoint?.startsWith('https://') || apiConfig.provider === 's3'; // sdk v3 only has signature support v4 const clientConfig = { forcePathStyle: apiConfig.s3ForcePathStyle === true ? true : false, // Use vhost style instead of path style - https://forums.aws.amazon.com/ann.jspa?annID=6776 region: apiConfig.region || 'us-east-1', credentials, - requestHandler, + requestHandler: new CustomHttpHandler({ + connectionTimeout: 60000, + socketTimeout: 20 * 60 * 1000, + removeExpectHeader: options.removeExpectHeader, + http: !isHttps, + acceptSelfSignedCerts: isHttps && (apiConfig.acceptSelfSignedCerts || apiConfig.bucket.includes('.')) + }), // logger: console }; if (options.retryStrategy) clientConfig.retryStrategy = options.retryStrategy; if (apiConfig.endpoint) clientConfig.endpoint = apiConfig.endpoint; - // s3 endpoint names come from the SDK - const isHttps = clientConfig.endpoint?.startsWith('https://') || apiConfig.provider === 's3'; - if (isHttps) { - if (apiConfig.acceptSelfSignedCerts || apiConfig.bucket.includes('.')) { - requestHandler.agent = new https.Agent({ rejectUnauthorized: false }); - } - } else { // http agent is required for http endpoints - requestHandler.agent = new http.Agent({}); - } - const client = constants.TEST ? new globalThis.S3Mock(clientConfig) : new S3(clientConfig); // https://github.com/aws/aws-sdk-js-v3/issues/6761#issuecomment-2574480834 // client.middlewareStack.add((next, context) => async (args) => { @@ -435,13 +449,11 @@ async function remove(apiConfig, filename) { const deleteParams = { Bucket: apiConfig.bucket, - Delete: { - Objects: [{ Key: filename }] - } + Key: filename }; - // deleteObjects does not return error if key is not found - const [error] = await safe(s3.deleteObjects(deleteParams)); + // deleteObject does not return error if key is not found + const [error] = await safe(s3.deleteObject(deleteParams)); if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, `Unable to remove ${filename}. ${formatError(error)}`); } @@ -466,7 +478,7 @@ async function removeDir(apiConfig, pathPrefix, progressCallback) { assert.strictEqual(typeof pathPrefix, 'string'); assert.strictEqual(typeof progressCallback, 'function'); - const s3 = createS3Client(apiConfig, { retryStrategy: RETRY_STRATEGY }); + const s3 = createS3Client(apiConfig, { retryStrategy: RETRY_STRATEGY, removeExpectHeader: true }); let total = 0; let marker = null;