diff --git a/package-lock.json b/package-lock.json index ce7850036..ceda21ef6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,11 +9,13 @@ "version": "1.0.0", "dependencies": { "@aws-sdk/client-route-53": "^3.744.0", + "@aws-sdk/client-s3": "^3.744.0", + "@aws-sdk/lib-storage": "^3.744.0", "@google-cloud/dns": "^4.0.0", "@google-cloud/storage": "^7.15.0", + "@smithy/node-http-handler": "^4.0.2", "@smithy/util-retry": "^4.0.1", "async": "^3.2.6", - "aws-sdk": "^2.1692.0", "basic-auth": "^2.0.1", "cloudron-manifestformat": "^5.26.2", "connect": "^3.7.0", @@ -69,7 +71,6 @@ "hock": "^1.4.1", "js2xmlparser": "^5.0.0", "mocha": "^11.1.0", - "mock-aws-s3": "github:cloudron-io/mock-aws-s3#0ad36e5ba", "nock": "^14.0.1", "ssh2": "^1.16.0", "yesno": "^0.4.0" @@ -103,6 +104,83 @@ "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", "license": "ISC" }, + "node_modules/@aws-crypto/crc32": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/crc32/-/crc32-5.2.0.tgz", + "integrity": "sha512-nLbCWqQNgUiwwtFsen1AdzAtvuLRsQS8rYgMuxCrdKf9kOssamGLuPwyTY9wyYblNr9+1XM8v6zoDTPPSIeANg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/util": "^5.2.0", + "@aws-sdk/types": "^3.222.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-crypto/crc32c": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/crc32c/-/crc32c-5.2.0.tgz", + "integrity": "sha512-+iWb8qaHLYKrNvGRbiYRHSdKRWhto5XlZUEBwDjYNf+ly5SVYG6zEoYIdxvf5R3zyeP16w4PLBn3rH1xc74Rag==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/util": "^5.2.0", + "@aws-sdk/types": "^3.222.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-crypto/sha1-browser": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/sha1-browser/-/sha1-browser-5.2.0.tgz", + "integrity": "sha512-OH6lveCFfcDjX4dbAvCFSYUjJZjDr/3XJ3xHtjn3Oj5b9RjojQo8npoLeA/bNwkOkrSQ0wgrHzXk4tDRxGKJeg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/supports-web-crypto": "^5.2.0", + "@aws-crypto/util": "^5.2.0", + "@aws-sdk/types": "^3.222.0", + "@aws-sdk/util-locate-window": "^3.0.0", + "@smithy/util-utf8": "^2.0.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-crypto/sha1-browser/node_modules/@smithy/is-array-buffer": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-2.2.0.tgz", + "integrity": "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-crypto/sha1-browser/node_modules/@smithy/util-buffer-from": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-2.2.0.tgz", + "integrity": "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/is-array-buffer": "^2.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-crypto/sha1-browser/node_modules/@smithy/util-utf8": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-2.3.0.tgz", + "integrity": "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/util-buffer-from": "^2.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/@aws-crypto/sha256-browser": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-browser/-/sha256-browser-5.2.0.tgz", @@ -281,6 +359,73 @@ "node": ">=18.0.0" } }, + "node_modules/@aws-sdk/client-s3": { + "version": "3.744.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-s3/-/client-s3-3.744.0.tgz", + "integrity": "sha512-UuiqxVI5FKlnNcWoDP8bsyJcMJa7XjGcCbVCfKSpSboNeBM4tQS3ZIViSYuz+BeO8/MuwCy7hKn7+Zjivit1nA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/sha1-browser": "5.2.0", + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "3.744.0", + "@aws-sdk/credential-provider-node": "3.744.0", + "@aws-sdk/middleware-bucket-endpoint": "3.734.0", + "@aws-sdk/middleware-expect-continue": "3.734.0", + "@aws-sdk/middleware-flexible-checksums": "3.744.0", + "@aws-sdk/middleware-host-header": "3.734.0", + "@aws-sdk/middleware-location-constraint": "3.734.0", + "@aws-sdk/middleware-logger": "3.734.0", + "@aws-sdk/middleware-recursion-detection": "3.734.0", + "@aws-sdk/middleware-sdk-s3": "3.744.0", + "@aws-sdk/middleware-ssec": "3.734.0", + "@aws-sdk/middleware-user-agent": "3.744.0", + "@aws-sdk/region-config-resolver": "3.734.0", + "@aws-sdk/signature-v4-multi-region": "3.744.0", + "@aws-sdk/types": "3.734.0", + "@aws-sdk/util-endpoints": "3.743.0", + "@aws-sdk/util-user-agent-browser": "3.734.0", + "@aws-sdk/util-user-agent-node": "3.744.0", + "@aws-sdk/xml-builder": "3.734.0", + "@smithy/config-resolver": "^4.0.1", + "@smithy/core": "^3.1.2", + "@smithy/eventstream-serde-browser": "^4.0.1", + "@smithy/eventstream-serde-config-resolver": "^4.0.1", + "@smithy/eventstream-serde-node": "^4.0.1", + "@smithy/fetch-http-handler": "^5.0.1", + "@smithy/hash-blob-browser": "^4.0.1", + "@smithy/hash-node": "^4.0.1", + "@smithy/hash-stream-node": "^4.0.1", + "@smithy/invalid-dependency": "^4.0.1", + "@smithy/md5-js": "^4.0.1", + "@smithy/middleware-content-length": "^4.0.1", + "@smithy/middleware-endpoint": "^4.0.3", + "@smithy/middleware-retry": "^4.0.4", + "@smithy/middleware-serde": "^4.0.2", + "@smithy/middleware-stack": "^4.0.1", + "@smithy/node-config-provider": "^4.0.1", + "@smithy/node-http-handler": "^4.0.2", + "@smithy/protocol-http": "^5.0.1", + "@smithy/smithy-client": "^4.1.3", + "@smithy/types": "^4.1.0", + "@smithy/url-parser": "^4.0.1", + "@smithy/util-base64": "^4.0.0", + "@smithy/util-body-length-browser": "^4.0.0", + "@smithy/util-body-length-node": "^4.0.0", + "@smithy/util-defaults-mode-browser": "^4.0.4", + "@smithy/util-defaults-mode-node": "^4.0.4", + "@smithy/util-endpoints": "^3.0.1", + "@smithy/util-middleware": "^4.0.1", + "@smithy/util-retry": "^4.0.1", + "@smithy/util-stream": "^4.0.2", + "@smithy/util-utf8": "^4.0.0", + "@smithy/util-waiter": "^4.0.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@aws-sdk/client-sso": { "version": "3.744.0", "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.744.0.tgz", @@ -511,6 +656,103 @@ "node": ">=18.0.0" } }, + "node_modules/@aws-sdk/lib-storage": { + "version": "3.744.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/lib-storage/-/lib-storage-3.744.0.tgz", + "integrity": "sha512-+WCIqTc2rT92kwL42di/zoHhQBGmUWAz8tr1wMQdTf+XlTDJZ4KhsfPwNsh5Gdge7il2yWuNRtB7uiZCFTM3kQ==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/abort-controller": "^4.0.1", + "@smithy/middleware-endpoint": "^4.0.3", + "@smithy/smithy-client": "^4.1.3", + "buffer": "5.6.0", + "events": "3.3.0", + "stream-browserify": "3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + }, + "peerDependencies": { + "@aws-sdk/client-s3": "^3.744.0" + } + }, + "node_modules/@aws-sdk/lib-storage/node_modules/buffer": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.6.0.tgz", + "integrity": "sha512-/gDYp/UtU0eA1ys8bOs9J6a+E/KWIY+DZ+Q2WESNUA0jFRsJOc0SNUO6xJ5SGA1xueg3NL65W6s+NY5l9cunuw==", + "license": "MIT", + "dependencies": { + "base64-js": "^1.0.2", + "ieee754": "^1.1.4" + } + }, + "node_modules/@aws-sdk/lib-storage/node_modules/events": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", + "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", + "license": "MIT", + "engines": { + "node": ">=0.8.x" + } + }, + "node_modules/@aws-sdk/middleware-bucket-endpoint": { + "version": "3.734.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-bucket-endpoint/-/middleware-bucket-endpoint-3.734.0.tgz", + "integrity": "sha512-etC7G18aF7KdZguW27GE/wpbrNmYLVT755EsFc8kXpZj8D6AFKxc7OuveinJmiy0bYXAMspJUWsF6CrGpOw6CQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "3.734.0", + "@aws-sdk/util-arn-parser": "3.723.0", + "@smithy/node-config-provider": "^4.0.1", + "@smithy/protocol-http": "^5.0.1", + "@smithy/types": "^4.1.0", + "@smithy/util-config-provider": "^4.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/middleware-expect-continue": { + "version": "3.734.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-expect-continue/-/middleware-expect-continue-3.734.0.tgz", + "integrity": "sha512-P38/v1l6HjuB2aFUewt7ueAW5IvKkFcv5dalPtbMGRhLeyivBOHwbCyuRKgVs7z7ClTpu9EaViEGki2jEQqEsQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "3.734.0", + "@smithy/protocol-http": "^5.0.1", + "@smithy/types": "^4.1.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/middleware-flexible-checksums": { + "version": "3.744.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-flexible-checksums/-/middleware-flexible-checksums-3.744.0.tgz", + "integrity": "sha512-4AuBdvkwfwagZQt3kt1b0x2dtC54cOrN5gt96V2b4wIjHBRxB/IfAyynahOgx3fd7Zjf74xwmxasjs7iJ8yglg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/crc32": "5.2.0", + "@aws-crypto/crc32c": "5.2.0", + "@aws-crypto/util": "5.2.0", + "@aws-sdk/core": "3.744.0", + "@aws-sdk/types": "3.734.0", + "@smithy/is-array-buffer": "^4.0.0", + "@smithy/node-config-provider": "^4.0.1", + "@smithy/protocol-http": "^5.0.1", + "@smithy/types": "^4.1.0", + "@smithy/util-middleware": "^4.0.1", + "@smithy/util-stream": "^4.0.2", + "@smithy/util-utf8": "^4.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@aws-sdk/middleware-host-header": { "version": "3.734.0", "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-host-header/-/middleware-host-header-3.734.0.tgz", @@ -526,6 +768,20 @@ "node": ">=18.0.0" } }, + "node_modules/@aws-sdk/middleware-location-constraint": { + "version": "3.734.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-location-constraint/-/middleware-location-constraint-3.734.0.tgz", + "integrity": "sha512-EJEIXwCQhto/cBfHdm3ZOeLxd2NlJD+X2F+ZTOxzokuhBtY0IONfC/91hOo5tWQweerojwshSMHRCKzRv1tlwg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "3.734.0", + "@smithy/types": "^4.1.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@aws-sdk/middleware-logger": { "version": "3.734.0", "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-logger/-/middleware-logger-3.734.0.tgz", @@ -569,6 +825,45 @@ "node": ">=18.0.0" } }, + "node_modules/@aws-sdk/middleware-sdk-s3": { + "version": "3.744.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-s3/-/middleware-sdk-s3-3.744.0.tgz", + "integrity": "sha512-zE0kNjMV7B8pC2ClhrV2gCj/gWLiinRkfPeiUevfjl+Hdke9zcAWVNHLeGV54FJjXQEdwIAjeE7WJdHo7hio7g==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "3.744.0", + "@aws-sdk/types": "3.734.0", + "@aws-sdk/util-arn-parser": "3.723.0", + "@smithy/core": "^3.1.2", + "@smithy/node-config-provider": "^4.0.1", + "@smithy/protocol-http": "^5.0.1", + "@smithy/signature-v4": "^5.0.1", + "@smithy/smithy-client": "^4.1.3", + "@smithy/types": "^4.1.0", + "@smithy/util-config-provider": "^4.0.0", + "@smithy/util-middleware": "^4.0.1", + "@smithy/util-stream": "^4.0.2", + "@smithy/util-utf8": "^4.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/middleware-ssec": { + "version": "3.734.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-ssec/-/middleware-ssec-3.734.0.tgz", + "integrity": "sha512-d4yd1RrPW/sspEXizq2NSOUivnheac6LPeLSLnaeTbBG9g1KqIqvCzP1TfXEqv2CrWfHEsWtJpX7oyjySSPvDQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "3.734.0", + "@smithy/types": "^4.1.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@aws-sdk/middleware-user-agent": { "version": "3.744.0", "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.744.0.tgz", @@ -653,6 +948,23 @@ "node": ">=18.0.0" } }, + "node_modules/@aws-sdk/signature-v4-multi-region": { + "version": "3.744.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.744.0.tgz", + "integrity": "sha512-QyrAevGGwceM+knGfV5r2NvSAjI94PETu6u+Fxalf8F/ybpK7qn1va0w3cGDU68oRqC0JHfo53JXjm9yQokj9Q==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/middleware-sdk-s3": "3.744.0", + "@aws-sdk/types": "3.734.0", + "@smithy/protocol-http": "^5.0.1", + "@smithy/signature-v4": "^5.0.1", + "@smithy/types": "^4.1.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@aws-sdk/token-providers": { "version": "3.744.0", "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.744.0.tgz", @@ -683,6 +995,18 @@ "node": ">=18.0.0" } }, + "node_modules/@aws-sdk/util-arn-parser": { + "version": "3.723.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-arn-parser/-/util-arn-parser-3.723.0.tgz", + "integrity": "sha512-ZhEfvUwNliOQROcAk34WJWVYTlTa4694kSVhDSjW6lE1bMataPnIN8A0ycukEzBXmd8ZSoBcQLn6lKGl7XIJ5w==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@aws-sdk/util-endpoints": { "version": "3.743.0", "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.743.0.tgz", @@ -1581,6 +1905,31 @@ "node": ">=18.0.0" } }, + "node_modules/@smithy/chunked-blob-reader": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@smithy/chunked-blob-reader/-/chunked-blob-reader-5.0.0.tgz", + "integrity": "sha512-+sKqDBQqb036hh4NPaUiEkYFkTUGYzRsn3EuFhyfQfMy6oGHEUJDurLP9Ufb5dasr/XiAmPNMr6wa9afjQB+Gw==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/chunked-blob-reader-native": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@smithy/chunked-blob-reader-native/-/chunked-blob-reader-native-4.0.0.tgz", + "integrity": "sha512-R9wM2yPmfEMsUmlMlIgSzOyICs0x9uu7UTHoccMyt7BWw8shcGM8HqB355+BZCPBcySvbTYMs62EgEQkNxz2ig==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/util-base64": "^4.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@smithy/config-resolver": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/@smithy/config-resolver/-/config-resolver-4.0.1.tgz", @@ -1632,6 +1981,76 @@ "node": ">=18.0.0" } }, + "node_modules/@smithy/eventstream-codec": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@smithy/eventstream-codec/-/eventstream-codec-4.0.1.tgz", + "integrity": "sha512-Q2bCAAR6zXNVtJgifsU16ZjKGqdw/DyecKNgIgi7dlqw04fqDu0mnq+JmGphqheypVc64CYq3azSuCpAdFk2+A==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/crc32": "5.2.0", + "@smithy/types": "^4.1.0", + "@smithy/util-hex-encoding": "^4.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/eventstream-serde-browser": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-browser/-/eventstream-serde-browser-4.0.1.tgz", + "integrity": "sha512-HbIybmz5rhNg+zxKiyVAnvdM3vkzjE6ccrJ620iPL8IXcJEntd3hnBl+ktMwIy12Te/kyrSbUb8UCdnUT4QEdA==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/eventstream-serde-universal": "^4.0.1", + "@smithy/types": "^4.1.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/eventstream-serde-config-resolver": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-config-resolver/-/eventstream-serde-config-resolver-4.0.1.tgz", + "integrity": "sha512-lSipaiq3rmHguHa3QFF4YcCM3VJOrY9oq2sow3qlhFY+nBSTF/nrO82MUQRPrxHQXA58J5G1UnU2WuJfi465BA==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.1.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/eventstream-serde-node": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-node/-/eventstream-serde-node-4.0.1.tgz", + "integrity": "sha512-o4CoOI6oYGYJ4zXo34U8X9szDe3oGjmHgsMGiZM0j4vtNoT+h80TLnkUcrLZR3+E6HIxqW+G+9WHAVfl0GXK0Q==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/eventstream-serde-universal": "^4.0.1", + "@smithy/types": "^4.1.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/eventstream-serde-universal": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-universal/-/eventstream-serde-universal-4.0.1.tgz", + "integrity": "sha512-Z94uZp0tGJuxds3iEAZBqGU2QiaBHP4YytLUjwZWx+oUeohCsLyUm33yp4MMBmhkuPqSbQCXq5hDet6JGUgHWA==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/eventstream-codec": "^4.0.1", + "@smithy/types": "^4.1.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@smithy/fetch-http-handler": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-5.0.1.tgz", @@ -1648,6 +2067,21 @@ "node": ">=18.0.0" } }, + "node_modules/@smithy/hash-blob-browser": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@smithy/hash-blob-browser/-/hash-blob-browser-4.0.1.tgz", + "integrity": "sha512-rkFIrQOKZGS6i1D3gKJ8skJ0RlXqDvb1IyAphksaFOMzkn3v3I1eJ8m7OkLj0jf1McP63rcCEoLlkAn/HjcTRw==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/chunked-blob-reader": "^5.0.0", + "@smithy/chunked-blob-reader-native": "^4.0.0", + "@smithy/types": "^4.1.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@smithy/hash-node": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/@smithy/hash-node/-/hash-node-4.0.1.tgz", @@ -1663,6 +2097,20 @@ "node": ">=18.0.0" } }, + "node_modules/@smithy/hash-stream-node": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@smithy/hash-stream-node/-/hash-stream-node-4.0.1.tgz", + "integrity": "sha512-U1rAE1fxmReCIr6D2o/4ROqAQX+GffZpyMt3d7njtGDr2pUNmAKRWa49gsNVhCh2vVAuf3wXzWwNr2YN8PAXIw==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.1.0", + "@smithy/util-utf8": "^4.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@smithy/invalid-dependency": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/@smithy/invalid-dependency/-/invalid-dependency-4.0.1.tgz", @@ -1688,6 +2136,20 @@ "node": ">=18.0.0" } }, + "node_modules/@smithy/md5-js": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@smithy/md5-js/-/md5-js-4.0.1.tgz", + "integrity": "sha512-HLZ647L27APi6zXkZlzSFZIjpo8po45YiyjMGJZM3gyDY8n7dPGdmxIIljLm4gPt/7rRvutLTTkYJpZVfG5r+A==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.1.0", + "@smithy/util-utf8": "^4.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@smithy/middleware-content-length": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/@smithy/middleware-content-length/-/middleware-content-length-4.0.1.tgz", @@ -2426,45 +2888,6 @@ "version": "0.4.0", "license": "MIT" }, - "node_modules/available-typed-arrays": { - "version": "1.0.5", - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/aws-sdk": { - "version": "2.1692.0", - "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.1692.0.tgz", - "integrity": "sha512-x511uiJ/57FIsbgUe5csJ13k3uzu25uWQE+XqfBis/sB0SFoiElJWXRkgEAUh0U6n40eT3ay5Ue4oPkRMu1LYw==", - "hasInstallScript": true, - "license": "Apache-2.0", - "dependencies": { - "buffer": "4.9.2", - "events": "1.1.1", - "ieee754": "1.1.13", - "jmespath": "0.16.0", - "querystring": "0.2.0", - "sax": "1.2.1", - "url": "0.10.3", - "util": "^0.12.4", - "uuid": "8.0.0", - "xml2js": "0.6.2" - }, - "engines": { - "node": ">= 10.0.0" - } - }, - "node_modules/aws-sdk/node_modules/uuid": { - "version": "8.0.0", - "license": "MIT", - "bin": { - "uuid": "dist/bin/uuid" - } - }, "node_modules/aws-ssl-profiles": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/aws-ssl-profiles/-/aws-ssl-profiles-1.1.2.tgz", @@ -2705,15 +3128,6 @@ "dev": true, "license": "ISC" }, - "node_modules/buffer": { - "version": "4.9.2", - "license": "MIT", - "dependencies": { - "base64-js": "^1.0.2", - "ieee754": "^1.1.4", - "isarray": "^1.0.0" - } - }, "node_modules/buffer-equal-constant-time": { "version": "1.0.1", "license": "BSD-3-Clause" @@ -2771,17 +3185,6 @@ "node": ">=14.16" } }, - "node_modules/call-bind": { - "version": "1.0.2", - "license": "MIT", - "dependencies": { - "function-bind": "^1.1.1", - "get-intrinsic": "^1.0.2" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/call-bind-apply-helpers": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.1.tgz", @@ -4115,13 +4518,6 @@ "node": ">=6" } }, - "node_modules/events": { - "version": "1.1.1", - "license": "MIT", - "engines": { - "node": ">=0.4.x" - } - }, "node_modules/expect.js": { "version": "0.3.1", "dev": true @@ -4472,13 +4868,6 @@ "integrity": "sha512-X8cqMLLie7KsNUDSdzeN8FYK9rEt4Dt67OsG/DNGnYTSDBG4uFAJFBnUeiV+zCVAvwFy56IjM9sH51jVaEhNxw==", "dev": true }, - "node_modules/for-each": { - "version": "0.3.3", - "license": "MIT", - "dependencies": { - "is-callable": "^1.1.3" - } - }, "node_modules/foreground-child": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.0.tgz", @@ -4551,29 +4940,6 @@ "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" }, - "node_modules/fs-extra": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-7.0.1.tgz", - "integrity": "sha512-YJDaCJZEnBmcbw13fvdAM9AwNOJwOzrE4pqMqBq5nFiEqXUqHwlK4B+3pUw6JNvfSPtX05xFHtYy/1ni01eGCw==", - "dev": true, - "dependencies": { - "graceful-fs": "^4.1.2", - "jsonfile": "^4.0.0", - "universalify": "^0.1.0" - }, - "engines": { - "node": ">=6 <7 || >=8" - } - }, - "node_modules/fs-extra/node_modules/universalify": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", - "integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==", - "dev": true, - "engines": { - "node": ">= 4.0.0" - } - }, "node_modules/fs.realpath": { "version": "1.0.0", "license": "ISC" @@ -4800,12 +5166,6 @@ "url": "https://github.com/sindresorhus/got?sponsor=1" } }, - "node_modules/graceful-fs": { - "version": "4.2.11", - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", - "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", - "dev": true - }, "node_modules/gtoken": { "version": "7.1.0", "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-7.1.0.tgz", @@ -5107,20 +5467,6 @@ "node": ">= 10" } }, - "node_modules/is-arguments": { - "version": "1.1.1", - "license": "MIT", - "dependencies": { - "call-bind": "^1.0.2", - "has-tostringtag": "^1.0.0" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/is-binary-path": { "version": "2.1.0", "dev": true, @@ -5132,16 +5478,6 @@ "node": ">=8" } }, - "node_modules/is-callable": { - "version": "1.2.7", - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/is-extglob": { "version": "2.1.1", "dev": true, @@ -5245,23 +5581,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/is-typed-array": { - "version": "1.1.10", - "license": "MIT", - "dependencies": { - "available-typed-arrays": "^1.0.5", - "call-bind": "^1.0.2", - "for-each": "^0.3.3", - "gopd": "^1.0.1", - "has-tostringtag": "^1.0.0" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/is-unicode-supported": { "version": "0.1.0", "dev": true, @@ -5320,13 +5639,6 @@ "node": ">=10" } }, - "node_modules/jmespath": { - "version": "0.16.0", - "license": "Apache-2.0", - "engines": { - "node": ">= 0.6.0" - } - }, "node_modules/jose": { "version": "5.9.6", "resolved": "https://registry.npmjs.org/jose/-/jose-5.9.6.tgz", @@ -5451,15 +5763,6 @@ "dev": true, "license": "ISC" }, - "node_modules/jsonfile": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz", - "integrity": "sha512-m6F1R3z8jjlf2imQHS2Qez5sjKWQzbuuhuJ/FKYFRZvPE3PuHcSMVZzfsLhGVOkfd20obL5SWEBew5ShlquNxg==", - "dev": true, - "optionalDependencies": { - "graceful-fs": "^4.1.6" - } - }, "node_modules/jsonwebtoken": { "version": "9.0.2", "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-9.0.2.tgz", @@ -6049,25 +6352,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/mock-aws-s3": { - "version": "4.0.2", - "resolved": "git+ssh://git@github.com/cloudron-io/mock-aws-s3.git#0ad36e5bae8d821921779012dd9f1a70397daca3", - "dev": true, - "dependencies": { - "bluebird": "^3.5.1", - "fs-extra": "^7.0.1", - "underscore": "1.12.1" - }, - "engines": { - "node": ">=10.0.0" - } - }, - "node_modules/mock-aws-s3/node_modules/underscore": { - "version": "1.12.1", - "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.12.1.tgz", - "integrity": "sha512-hEQt0+ZLDVUMhebKxL4x1BTtDY7bavVofhZ9KZ4aI26X9SRaE+Y3m83XUL1UP2jn8ynjndwCCpEHdUG+9pP1Tw==", - "dev": true - }, "node_modules/moment": { "version": "2.30.1", "resolved": "https://registry.npmjs.org/moment/-/moment-2.30.1.tgz", @@ -6798,10 +7082,6 @@ "once": "^1.3.1" } }, - "node_modules/punycode": { - "version": "1.3.2", - "license": "MIT" - }, "node_modules/qrcode": { "version": "1.5.4", "resolved": "https://registry.npmjs.org/qrcode/-/qrcode-1.5.4.tgz", @@ -6890,12 +7170,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/querystring": { - "version": "0.2.0", - "engines": { - "node": ">=0.4.x" - } - }, "node_modules/queue-tick": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/queue-tick/-/queue-tick-1.0.1.tgz", @@ -7475,6 +7749,30 @@ "node": ">= 0.6" } }, + "node_modules/stream-browserify": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/stream-browserify/-/stream-browserify-3.0.0.tgz", + "integrity": "sha512-H73RAHsVBapbim0tU2JwwOiXUj+fikfiaoYAKHF3VJfA0pe2BCzkhAHBlLG6REzE+2WNZcxOXjK7lkso+9euLA==", + "license": "MIT", + "dependencies": { + "inherits": "~2.0.4", + "readable-stream": "^3.5.0" + } + }, + "node_modules/stream-browserify/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/stream-events": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/stream-events/-/stream-events-1.0.5.tgz", @@ -7982,14 +8280,6 @@ "node": ">=6" } }, - "node_modules/url": { - "version": "0.10.3", - "license": "MIT", - "dependencies": { - "punycode": "1.3.2", - "querystring": "0.2.0" - } - }, "node_modules/url-equal": { "version": "0.1.2-1", "dev": true, @@ -8003,17 +8293,6 @@ "dev": true, "license": "MIT" }, - "node_modules/util": { - "version": "0.12.5", - "license": "MIT", - "dependencies": { - "inherits": "^2.0.3", - "is-arguments": "^1.0.4", - "is-generator-function": "^1.0.7", - "is-typed-array": "^1.1.3", - "which-typed-array": "^1.1.2" - } - }, "node_modules/util-deprecate": { "version": "1.0.2", "license": "MIT" @@ -8198,24 +8477,6 @@ "version": "2.0.0", "license": "ISC" }, - "node_modules/which-typed-array": { - "version": "1.1.9", - "license": "MIT", - "dependencies": { - "available-typed-arrays": "^1.0.5", - "call-bind": "^1.0.2", - "for-each": "^0.3.3", - "gopd": "^1.0.1", - "has-tostringtag": "^1.0.0", - "is-typed-array": "^1.1.10" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/winston": { "version": "2.4.5", "license": "MIT", diff --git a/package.json b/package.json index b3b1d7988..6c3be2a25 100644 --- a/package.json +++ b/package.json @@ -17,11 +17,13 @@ }, "dependencies": { "@aws-sdk/client-route-53": "^3.744.0", + "@aws-sdk/client-s3": "^3.744.0", + "@aws-sdk/lib-storage": "^3.744.0", "@google-cloud/dns": "^4.0.0", "@google-cloud/storage": "^7.15.0", + "@smithy/node-http-handler": "^4.0.2", "@smithy/util-retry": "^4.0.1", "async": "^3.2.6", - "aws-sdk": "^2.1692.0", "basic-auth": "^2.0.1", "cloudron-manifestformat": "^5.26.2", "connect": "^3.7.0", @@ -73,7 +75,6 @@ "hock": "^1.4.1", "js2xmlparser": "^5.0.0", "mocha": "^11.1.0", - "mock-aws-s3": "github:cloudron-io/mock-aws-s3#0ad36e5ba", "nock": "^14.0.1", "ssh2": "^1.16.0", "yesno": "^0.4.0" diff --git a/setup/start/systemd/box.service b/setup/start/systemd/box.service index cc8779d3a..0e0fc30a2 100644 --- a/setup/start/systemd/box.service +++ b/setup/start/systemd/box.service @@ -15,7 +15,7 @@ ExecStart=/home/yellowtent/box/box.js ExecReload=/bin/kill -HUP $MAINPID ; we run commands like df which will parse properly only with correct locale ; add "oidc-provider:*" to DEBUG for OpenID debugging -Environment="HOME=/home/yellowtent" "USER=yellowtent" "DEBUG=box:*,connect-lastmile,-box:ldap,-box:oidc" "BOX_ENV=cloudron" "NODE_ENV=production" "LC_ALL=C" "AWS_SDK_JS_SUPPRESS_MAINTENANCE_MODE_MESSAGE=1" +Environment="HOME=/home/yellowtent" "USER=yellowtent" "DEBUG=box:*,connect-lastmile,-box:ldap,-box:oidc" "BOX_ENV=cloudron" "NODE_ENV=production" "LC_ALL=C" ; kill apptask processes as well KillMode=control-group ; Do not kill this process on OOM. Children inherit this score. Do not set it to -1000 so that MemoryMax can keep working diff --git a/src/scripts/starttask.sh b/src/scripts/starttask.sh index e01e246cc..512eccdf8 100755 --- a/src/scripts/starttask.sh +++ b/src/scripts/starttask.sh @@ -46,7 +46,7 @@ fi # DEBUG has to be hardcoded because it is not set in the tests. --setenv is required for ubuntu 16 (-E does not work) # NODE_OPTIONS is used because env -S does not work in ubuntu 16/18. # it seems systemd-run does not return the exit status of the process despite --wait -if ! systemd-run --unit "${service_name}" --nice "${nice}" --uid=${id} --gid=${id} ${options} --setenv HOME=${HOME} --setenv USER=${SUDO_USER} --setenv DEBUG=box:* --setenv BOX_ENV=${BOX_ENV} --setenv NODE_ENV=production --setenv NODE_OPTIONS=--unhandled-rejections=strict --setenv AWS_SDK_JS_SUPPRESS_MAINTENANCE_MODE_MESSAGE=1 "${task_worker}" "${task_id}" "${logfile}"; then +if ! systemd-run --unit "${service_name}" --nice "${nice}" --uid=${id} --gid=${id} ${options} --setenv HOME=${HOME} --setenv USER=${SUDO_USER} --setenv DEBUG=box:* --setenv BOX_ENV=${BOX_ENV} --setenv NODE_ENV=production --setenv NODE_OPTIONS=--unhandled-rejections=strict --setenv "${task_worker}" "${task_id}" "${logfile}"; then echo "Service ${service_name} failed to run" # this only happens if the path to task worker itself is wrong fi diff --git a/src/storage/gcs.js b/src/storage/gcs.js index 10ad5e2ed..fcaddfcf4 100644 --- a/src/storage/gcs.js +++ b/src/storage/gcs.js @@ -16,10 +16,6 @@ exports = module.exports = { testConfig, removePrivateFields, injectPrivateFields, - - // Used to mock GCS - _mockInject: mockInject, - _mockRestore: mockRestore }; const assert = require('assert'), @@ -27,23 +23,10 @@ const assert = require('assert'), BoxError = require('../boxerror.js'), constants = require('../constants.js'), debug = require('debug')('box:storage/gcs'), + GCS = require('@google-cloud/storage').Storage, path = require('path'), safe = require('safetydance'); -let GCS = require('@google-cloud/storage').Storage; - -// test only -let originalGCS; -function mockInject(mock) { - originalGCS = GCS; - GCS = mock; -} - -function mockRestore() { - GCS = originalGCS; -} - -// internal only function getBucket(apiConfig) { assert.strictEqual(typeof apiConfig, 'object'); @@ -56,7 +39,8 @@ function getBucket(apiConfig) { } }; - return new GCS(gcsConfig).bucket(apiConfig.bucket); + const gcs = constants.TEST ? new globalThis.GCSMock(gcsConfig) : new GCS(gcsConfig); + return gcs.bucket(apiConfig.bucket); } async function getAvailableSize(apiConfig) { @@ -143,17 +127,13 @@ async function copy(apiConfig, oldFilePath, newFilePath, progressCallback) { assert.strictEqual(typeof newFilePath, 'string'); assert.strictEqual(typeof progressCallback, 'function'); - function copyFile(entry, iteratorCallback) { - var relativePath = path.relative(oldFilePath, entry.fullPath); + async function copyFile(entry) { + const relativePath = path.relative(oldFilePath, entry.fullPath); - getBucket(apiConfig).file(entry.fullPath).copy(path.join(newFilePath, relativePath), function(error) { - if (error) debug('copyBackup: gcs copy error. %o', error); - - if (error && error.code === 404) return iteratorCallback(new BoxError(BoxError.NOT_FOUND, 'Old backup not found')); - if (error) return iteratorCallback(new BoxError(BoxError.EXTERNAL_ERROR, error.message)); - - iteratorCallback(null); - }); + const [copyError] = await safe(getBucket(apiConfig).file(entry.fullPath).copy(path.join(newFilePath, relativePath))); + if (copyError) debug('copyBackup: gcs copy error. %o', copyError); + if (copyError && copyError.code === 404) throw new BoxError(BoxError.NOT_FOUND, 'Old backup not found'); + if (copyError) throw new BoxError(BoxError.EXTERNAL_ERROR, copyError.message); } const batchSize = 1000; diff --git a/src/storage/s3.js b/src/storage/s3.js index 4da35be88..c8de6a852 100644 --- a/src/storage/s3.js +++ b/src/storage/s3.js @@ -18,76 +18,67 @@ exports = module.exports = { injectPrivateFields, // Used to mock AWS - _mockInject: mockInject, - _mockRestore: mockRestore, _chunk: chunk }; -// https://github.com/aws/aws-sdk-js/issues/4354 -require('aws-sdk/lib/maintenance_mode_message').suppress = true; - const assert = require('assert'), async = require('async'), - AwsSdk = require('aws-sdk'), BoxError = require('../boxerror.js'), + { ConfiguredRetryStrategy } = require('@smithy/util-retry'), constants = require('../constants.js'), debug = require('debug')('box:storage/s3'), + http = require('http'), https = require('https'), + { NodeHttpHandler } = require('@smithy/node-http-handler'), { PassThrough } = require('node:stream'), path = require('path'), Readable = require('stream').Readable, + { S3 } = require('@aws-sdk/client-s3'), safe = require('safetydance'), - _ = require('underscore'); - -let aws = AwsSdk; - -// test only -let originalAWS; -function mockInject(mock) { - originalAWS = aws; - aws = mock; -} - -function mockRestore() { - aws = originalAWS; -} + { Upload } = require('@aws-sdk/lib-storage'); function S3_NOT_FOUND(error) { return error.code === 'NoSuchKey' || error.code === 'NotFound' || error.code === 'ENOENT'; } -function getS3Config(apiConfig) { +const RETRY_STRATEGY = new ConfiguredRetryStrategy(10 /* max attempts */, (/* attempt */) => 20000 /* constant backoff */); + +function createS3Client(apiConfig, options) { assert.strictEqual(typeof apiConfig, 'object'); + assert.strictEqual(typeof options, 'object'); const credentials = { - signatureVersion: apiConfig.signatureVersion || 'v4', - s3ForcePathStyle: false, // Use vhost style instead of path style - https://forums.aws.amazon.com/ann.jspa?annID=6776 accessKeyId: apiConfig.accessKeyId, - secretAccessKey: apiConfig.secretAccessKey, - region: apiConfig.region || 'us-east-1', - maxRetries: 10, - retryDelayOptions: { - customBackoff: (/* retryCount, error */) => 20000 // constant backoff - https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/Config.html#retryDelayOptions-property - }, - httpOptions: { - connectTimeout: 60000, // https://github.com/aws/aws-sdk-js/pull/1446 - timeout: 20 * 60 * 1000 // https://github.com/aws/aws-sdk-js/issues/1704 - } + secretAccessKey: apiConfig.secretAccessKey }; - if (apiConfig.endpoint) credentials.endpoint = apiConfig.endpoint; + const requestHandler = new NodeHttpHandler({ + connectionTimeout: 60000, + socketTimeout: 20 * 60 * 1000 + }); - if (apiConfig.s3ForcePathStyle === true) credentials.s3ForcePathStyle = true; + // sdk v3 only has signature support v4 + const clientConfig = { + forcePathStyle: apiConfig.s3ForcePathStyle === true ? true : false, // Use vhost style instead of path style - https://forums.aws.amazon.com/ann.jspa?annID=6776 + region: apiConfig.region || 'us-east-1', + credentials, + requestHandler: requestHandler + }; + + if (options.retryStrategy) clientConfig.retryStrategy = options.retryStrategy; + if (apiConfig.endpoint) clientConfig.endpoint = apiConfig.endpoint; // s3 endpoint names come from the SDK - const isHttps = (credentials.endpoint && credentials.endpoint.startsWith('https://')) || apiConfig.provider === 's3'; - if (isHttps) { // only set agent for https calls. otherwise, it crashes + const isHttps = clientConfig.endpoint?.startsWith('https://') || apiConfig.provider === 's3'; + if (isHttps) { if (apiConfig.acceptSelfSignedCerts || apiConfig.bucket.includes('.')) { - credentials.httpOptions.agent = new https.Agent({ rejectUnauthorized: false }); + requestHandler.agent = new https.Agent({ rejectUnauthorized: false }); } + } else { // http agent is required for http endpoints + requestHandler.agent = new http.Agent({}); } - return credentials; + return constants.TEST ? new globalThis.S3Mock(clientConfig) : new S3(clientConfig); } async function getAvailableSize(apiConfig) { @@ -100,8 +91,7 @@ async function upload(apiConfig, backupFilePath) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof backupFilePath, 'string'); - const credentials = getS3Config(apiConfig); - const s3 = new aws.S3(credentials); + const s3 = createS3Client(apiConfig, { retryStrategy: RETRY_STRATEGY }); // s3.upload automatically does a multi-part upload. we set queueSize to 3 to reduce memory usage // uploader will buffer at most queueSize * partSize bytes into memory at any given time. @@ -111,15 +101,21 @@ async function upload(apiConfig, backupFilePath) { const passThrough = new PassThrough(); - const params = { - Bucket: apiConfig.bucket, - Key: backupFilePath, - Body: passThrough + const options = { + client: s3, + params: { + Bucket: apiConfig.bucket, + Key: backupFilePath, + Body: passThrough + }, + partSize, + queueSize: 3, + leavePartsOnError: false }; - const managedUpload = s3.upload(params, { partSize, queueSize: 3 }); + const managedUpload = constants.TEST ? new globalThis.S3MockUpload(options) : new Upload(options); managedUpload.on('httpUploadProgress', (progress) => debug(`Upload progress: ${JSON.stringify(progress)}`)); - const uploadPromise = managedUpload.promise(); + const uploadPromise = managedUpload.done(); return { stream: passThrough, @@ -135,9 +131,7 @@ async function exists(apiConfig, backupFilePath) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof backupFilePath, 'string'); - const credentials = getS3Config(apiConfig); - - const s3 = new aws.S3(_.omit(credentials, 'retryDelayOptions', 'maxRetries')); + const s3 = createS3Client(apiConfig, { retryStrategy: null }); if (!backupFilePath.endsWith('/')) { // check for file const params = { @@ -145,7 +139,7 @@ async function exists(apiConfig, backupFilePath) { Key: backupFilePath }; - const [error, response] = await safe(s3.headObject(params).promise()); + const [error, response] = await safe(s3.headObject(params)); if (error && S3_NOT_FOUND(error)) return false; if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, `Error headObject ${backupFilePath}. Message: ${error.message} HTTP Code: ${error.code}`); if (!response || typeof response.Metadata !== 'object') throw new BoxError(BoxError.EXTERNAL_ERROR, 'not a s3 endpoint'); @@ -158,7 +152,7 @@ async function exists(apiConfig, backupFilePath) { MaxKeys: 1 }; - const [error, listData] = await safe(s3.listObjects(listParams).promise()); + const [error, listData] = await safe(s3.listObjects(listParams)); if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, `Error listing objects ${backupFilePath}. Message: ${error.message} HTTP Code: ${error.code}`); return listData.Contents.length !== 0; @@ -193,24 +187,23 @@ class S3MultipartDownloadStream extends Readable { } } - _downloadRange(offset, length) { + async _downloadRange(offset, length) { const params = Object.assign({}, this._params); const lastPos = offset + length - 1; const range = `bytes=${offset}-${lastPos}`; params['Range'] = range; - this._s3.getObject(params, (error, data) => { - if (error) return this._handleError(error); + const [error, data] = await safe(this._s3.getObject(params)); + if (error) return this._handleError(error); - const length = parseInt(data.ContentLength, 10); + const contentLength = parseInt(data.ContentLength, 10); // should be same as length - if (length > 0) { - this._readSize += length; - this.push(data.Body); - } else { - this._done(); - } - }); + if (contentLength > 0) { + this._readSize += contentLength; + this.push(data.Body); + } else { + this._done(); + } } _nextDownload() { @@ -223,22 +216,21 @@ class S3MultipartDownloadStream extends Readable { this._downloadRange(this._readSize, len); } - _fetchSize() { - this._s3.headObject(this._params, (error, data) => { - if (error) return this._handleError(error); + async _fetchSize() { + const [error, data] = await safe(this._s3.headObject(this._params)); + if (error) return this._handleError(error); - const length = parseInt(data.ContentLength, 10); + const length = parseInt(data.ContentLength, 10); - if (length > 0) { - this._fileSize = length; - this._nextDownload(); - } else { - this._done(); - } - }); + if (length > 0) { + this._fileSize = length; + this._nextDownload(); + } else { + this._done(); + } } - _read() { + _read() { // reimp if (this._readSize === this._fileSize) return this._done(); if (this._readSize === 0) return this._fetchSize(); @@ -250,14 +242,12 @@ async function download(apiConfig, backupFilePath) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof backupFilePath, 'string'); - const credentials = getS3Config(apiConfig); - const params = { Bucket: apiConfig.bucket, Key: backupFilePath }; - const s3 = new aws.S3(credentials); + const s3 = createS3Client(apiConfig, { retryStrategy: RETRY_STRATEGY }); return new S3MultipartDownloadStream(s3, params, { blockSize: 64 * 1024 * 1024 }); } @@ -267,9 +257,7 @@ async function listDir(apiConfig, dir, batchSize, marker) { assert.strictEqual(typeof batchSize, 'number'); assert(typeof marker !== 'undefined'); - const credentials = getS3Config(apiConfig); - - const s3 = new aws.S3(credentials); + const s3 = createS3Client(apiConfig, { retryStrategy: RETRY_STRATEGY }); const listParams = { Bucket: apiConfig.bucket, Prefix: dir, @@ -277,7 +265,7 @@ async function listDir(apiConfig, dir, batchSize, marker) { }; if (marker) listParams.Marker = marker; - const [error, listData] = await safe(s3.listObjects(listParams).promise()); + const [error, listData] = await safe(s3.listObjects(listParams)); if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, `Error listing objects in ${dir}. Message: ${error.message} HTTP Code: ${error.code}`); if (listData.Contents.length === 0) return { entries: [], marker: null }; // no more const entries = listData.Contents.map(function (c) { return { fullPath: c.Key, size: c.Size }; }); @@ -297,115 +285,107 @@ function encodeCopySource(bucket, path) { return `/${bucket}/${output}`; } +async function copyFile(apiConfig, oldFilePath, newFilePath, entry, progressCallback) { + assert.strictEqual(typeof apiConfig, 'object'); + assert.strictEqual(typeof oldFilePath, 'string'); + assert.strictEqual(typeof newFilePath, 'string'); + assert.strictEqual(typeof entry, 'object'); + assert.strictEqual(typeof progressCallback, 'function'); + + const s3 = createS3Client(apiConfig, { retryStrategy: RETRY_STRATEGY }); // https://docs.aws.amazon.com/sdkref/latest/guide/feature-retry-behavior.html + const relativePath = path.relative(oldFilePath, entry.fullPath); + + function throwError(error) { + if (error) debug(`copy: s3 copy error when copying ${entry.fullPath}: ${error}`); + + if (error && S3_NOT_FOUND(error)) throw new BoxError(BoxError.NOT_FOUND, `Old backup not found: ${entry.fullPath}`); + if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, `Error copying ${entry.fullPath} (${entry.size} bytes): ${error.code || ''} ${error}`); + } + + const copyParams = { + Bucket: apiConfig.bucket, + Key: path.join(newFilePath, relativePath) + }; + + // S3 copyObject has a file size limit of 5GB so if we have larger files, we do a multipart copy + const largeFileLimit = (apiConfig.provider === 'vultr-objectstorage' || apiConfig.provider === 'exoscale-sos' || apiConfig.provider === 'backblaze-b2' || apiConfig.provider === 'digitalocean-spaces') ? 1024 * 1024 * 1024 : 3 * 1024 * 1024 * 1024; + + if (entry.size < largeFileLimit) { + progressCallback({ message: `Copying ${relativePath || oldFilePath}` }); + + copyParams.CopySource = encodeCopySource(apiConfig.bucket, entry.fullPath); + const [copyError] = await safe(s3.copyObject(copyParams)); + if (copyError) return throwError(copyError); + return; + } + + progressCallback({ message: `Copying (multipart) ${relativePath || oldFilePath}` }); + + const [createMultipartError, multipart] = await safe(s3.createMultipartUpload(copyParams)); + if (createMultipartError) return throwError(createMultipartError); + + // Exoscale (96M) was suggested by exoscale. 1GB for others is arbitrary size + const chunkSize = apiConfig.provider === 'exoscale-sos' ? 96 * 1024 * 1024 : 1024 * 1024 * 1024; + const uploadId = multipart.UploadId; + const uploadedParts = [], ranges = []; + + let cur = 0; + while (cur + chunkSize < entry.size) { + ranges.push({ startBytes: cur, endBytes: cur + chunkSize - 1 }); + cur += chunkSize; + } + ranges.push({ startBytes: cur, endBytes: entry.size-1 }); + + const [copyError] = await safe(async.eachOfLimit(ranges, 3, async function copyChunk(range, index) { + const partCopyParams = { + Bucket: apiConfig.bucket, + Key: path.join(newFilePath, relativePath), + CopySource: encodeCopySource(apiConfig.bucket, entry.fullPath), // See aws-sdk-js/issues/1302 + CopySourceRange: 'bytes=' + range.startBytes + '-' + range.endBytes, + PartNumber: index+1, + UploadId: uploadId + }; + + progressCallback({ message: `Copying part ${partCopyParams.PartNumber} - ${partCopyParams.CopySource} ${partCopyParams.CopySourceRange}` }); + + const part = await s3.uploadPartCopy(partCopyParams); + progressCallback({ message: `Copied part ${partCopyParams.PartNumber} - Etag: ${part.CopyPartResult.ETag}` }); + + if (!part.CopyPartResult.ETag) throw new Error('Multi-part copy is broken or not implemented by the S3 storage provider'); + + uploadedParts[index] = { ETag: part.CopyPartResult.ETag, PartNumber: partCopyParams.PartNumber }; + })); + + if (copyError) { // we must still recommend the user to set a AbortIncompleteMultipartUpload lifecycle rule + const abortParams = { + Bucket: apiConfig.bucket, + Key: path.join(newFilePath, relativePath), + UploadId: uploadId + }; + progressCallback({ message: `Aborting multipart copy of ${relativePath || oldFilePath}` }); + await safe(s3.abortMultipartUpload(abortParams), { debug }); // ignore any abort errors + return throwError(copyError); + } + + const completeMultipartParams = { + Bucket: apiConfig.bucket, + Key: path.join(newFilePath, relativePath), + MultipartUpload: { Parts: uploadedParts }, + UploadId: uploadId + }; + + progressCallback({ message: `Finishing multipart copy - ${completeMultipartParams.Key}` }); + + const [completeMultipartError] = await safe(s3.completeMultipartUpload(completeMultipartParams)); + if (completeMultipartError) return throwError(completeMultipartError); +} + async function copy(apiConfig, oldFilePath, newFilePath, progressCallback) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof oldFilePath, 'string'); assert.strictEqual(typeof newFilePath, 'string'); assert.strictEqual(typeof progressCallback, 'function'); - function copyFile(entry, iteratorCallback) { - const credentials = getS3Config(apiConfig); - - const s3 = new aws.S3(credentials); - const relativePath = path.relative(oldFilePath, entry.fullPath); - - function done(error) { - if (error) debug(`copy: s3 copy error when copying ${entry.fullPath}: ${error}`); - - if (error && S3_NOT_FOUND(error)) return iteratorCallback(new BoxError(BoxError.NOT_FOUND, `Old backup not found: ${entry.fullPath}`)); - if (error) return iteratorCallback(new BoxError(BoxError.EXTERNAL_ERROR, `Error copying ${entry.fullPath} (${entry.size} bytes): ${error.code || ''} ${error}`)); - - iteratorCallback(null); - } - - const copyParams = { - Bucket: apiConfig.bucket, - Key: path.join(newFilePath, relativePath) - }; - - // S3 copyObject has a file size limit of 5GB so if we have larger files, we do a multipart copy - const largeFileLimit = (apiConfig.provider === 'vultr-objectstorage' || apiConfig.provider === 'exoscale-sos' || apiConfig.provider === 'backblaze-b2' || apiConfig.provider === 'digitalocean-spaces') ? 1024 * 1024 * 1024 : 3 * 1024 * 1024 * 1024; - - if (entry.size < largeFileLimit) { - progressCallback({ message: `Copying ${relativePath || oldFilePath}` }); - - copyParams.CopySource = encodeCopySource(apiConfig.bucket, entry.fullPath); - s3.copyObject(copyParams, done).on('retry', function (response) { - progressCallback({ message: `Retrying (${response.retryCount+1}) copy of ${relativePath || oldFilePath}. Error: ${response.error} ${response.httpResponse.statusCode}` }); - // on DO, we get a random 408. these are not retried by the SDK - if (response.error) response.error.retryable = true; // https://github.com/aws/aws-sdk-js/issues/412 - }); - - return; - } - - progressCallback({ message: `Copying (multipart) ${relativePath || oldFilePath}` }); - - s3.createMultipartUpload(copyParams, function (error, multipart) { - if (error) return done(error); - - // Exoscale (96M) was suggested by exoscale. 1GB for others is arbitrary size - const chunkSize = apiConfig.provider === 'exoscale-sos' ? 96 * 1024 * 1024 : 1024 * 1024 * 1024; - const uploadId = multipart.UploadId; - const uploadedParts = [], ranges = []; - - let cur = 0; - while (cur + chunkSize < entry.size) { - ranges.push({ startBytes: cur, endBytes: cur + chunkSize - 1 }); - cur += chunkSize; - } - ranges.push({ startBytes: cur, endBytes: entry.size-1 }); - - async.eachOfLimit(ranges, 3, function copyChunk(range, index, iteratorDone) { - const partCopyParams = { - Bucket: apiConfig.bucket, - Key: path.join(newFilePath, relativePath), - CopySource: encodeCopySource(apiConfig.bucket, entry.fullPath), // See aws-sdk-js/issues/1302 - CopySourceRange: 'bytes=' + range.startBytes + '-' + range.endBytes, - PartNumber: index+1, - UploadId: uploadId - }; - - progressCallback({ message: `Copying part ${partCopyParams.PartNumber} - ${partCopyParams.CopySource} ${partCopyParams.CopySourceRange}` }); - - s3.uploadPartCopy(partCopyParams, function (error, part) { - if (error) return iteratorDone(error); - - progressCallback({ message: `Copying part ${partCopyParams.PartNumber} - Etag: ${part.CopyPartResult.ETag}` }); - - if (!part.CopyPartResult.ETag) return iteratorDone(new Error('Multi-part copy is broken or not implemented by the S3 storage provider')); - - uploadedParts[index] = { ETag: part.CopyPartResult.ETag, PartNumber: partCopyParams.PartNumber }; - - iteratorDone(); - }).on('retry', function (response) { - progressCallback({ message: `Retrying (${response.retryCount+1}) multipart copy of ${relativePath || oldFilePath}. Error: ${response.error} ${response.httpResponse.statusCode}` }); - }); - }, function chunksCopied(error) { - if (error) { // we must still recommend the user to set a AbortIncompleteMultipartUpload lifecycle rule - const abortParams = { - Bucket: apiConfig.bucket, - Key: path.join(newFilePath, relativePath), - UploadId: uploadId - }; - progressCallback({ message: `Aborting multipart copy of ${relativePath || oldFilePath}` }); - return s3.abortMultipartUpload(abortParams, () => done(error)); // ignore any abort errors - } - - const completeMultipartParams = { - Bucket: apiConfig.bucket, - Key: path.join(newFilePath, relativePath), - MultipartUpload: { Parts: uploadedParts }, - UploadId: uploadId - }; - - progressCallback({ message: `Finishing multipart copy - ${completeMultipartParams.Key}` }); - - s3.completeMultipartUpload(completeMultipartParams, done); - }); - }); - } - let total = 0; const concurrency = apiConfig.limits?.copyConcurrency || (apiConfig.provider === 's3' ? 500 : 10); progressCallback({ message: `Copying with concurrency of ${concurrency}` }); @@ -415,7 +395,7 @@ async function copy(apiConfig, oldFilePath, newFilePath, progressCallback) { const batch = await listDir(apiConfig, oldFilePath, 1000, marker); total += batch.entries.length; progressCallback({ message: `Copying files from ${total-batch.entries.length}-${total}` }); - await async.eachLimit(batch.entries, concurrency, copyFile); + await async.eachLimit(batch.entries, concurrency, async (entry) => await copyFile(apiConfig, oldFilePath, newFilePath, entry, progressCallback)); if (!batch.marker) break; marker = batch.marker; } @@ -427,9 +407,7 @@ async function remove(apiConfig, filename) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof filename, 'string'); - const credentials = getS3Config(apiConfig); - - const s3 = new aws.S3(credentials); + const s3 = createS3Client(apiConfig, { retryStrategy: RETRY_STRATEGY }); const deleteParams = { Bucket: apiConfig.bucket, @@ -439,7 +417,7 @@ async function remove(apiConfig, filename) { }; // deleteObjects does not return error if key is not found - const [error] = await safe(s3.deleteObjects(deleteParams).promise()); + const [error] = await safe(s3.deleteObjects(deleteParams)); if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, `Unable to remove ${deleteParams.Key}. error: ${error.message}`); } @@ -464,8 +442,7 @@ async function removeDir(apiConfig, pathPrefix, progressCallback) { assert.strictEqual(typeof pathPrefix, 'string'); assert.strictEqual(typeof progressCallback, 'function'); - const credentials = getS3Config(apiConfig); - const s3 = new aws.S3(credentials); + const s3 = createS3Client(apiConfig, { retryStrategy: RETRY_STRATEGY }); let total = 0; let marker = null; @@ -489,7 +466,7 @@ async function removeDir(apiConfig, pathPrefix, progressCallback) { progressCallback({ message: `Removing ${objects.length} files from ${objects[0].fullPath} to ${objects[objects.length-1].fullPath}` }); // deleteObjects does not return error if key is not found - const [error] = await safe(s3.deleteObjects(deleteParams).promise()); + const [error] = await safe(s3.deleteObjects(deleteParams)); if (error) { progressCallback({ message: `Unable to remove ${deleteParams.Key} ${error.message || error.code}` }); throw new BoxError(BoxError.EXTERNAL_ERROR, `Unable to remove ${deleteParams.Key}. error: ${error.message}`); @@ -523,17 +500,14 @@ async function testConfig(apiConfig) { if ('acceptSelfSignedCerts' in apiConfig && typeof apiConfig.acceptSelfSignedCerts !== 'boolean') throw new BoxError(BoxError.BAD_FIELD, 'acceptSelfSignedCerts must be a boolean'); if ('s3ForcePathStyle' in apiConfig && typeof apiConfig.s3ForcePathStyle !== 'boolean') throw new BoxError(BoxError.BAD_FIELD, 's3ForcePathStyle must be a boolean'); - // attempt to upload and delete a file with new credentials - const credentials = getS3Config(apiConfig); - const putParams = { Bucket: apiConfig.bucket, Key: path.join(apiConfig.prefix, 'snapshot/cloudron-testfile'), Body: 'testcontent' }; - const s3 = new aws.S3(_.omit(credentials, 'retryDelayOptions', 'maxRetries')); - const [putError] = await safe(s3.putObject(putParams).promise()); + const s3 = createS3Client(apiConfig, {}); + const [putError] = await safe(s3.putObject(putParams)); if (putError) throw new BoxError(BoxError.EXTERNAL_ERROR, `Error put object cloudron-testfile. Message: ${putError.message} HTTP Code: ${putError.code}`); const listParams = { @@ -542,7 +516,7 @@ async function testConfig(apiConfig) { MaxKeys: 1 }; - const [listError] = await safe(s3.listObjects(listParams).promise()); + const [listError] = await safe(s3.listObjects(listParams)); if (listError) throw new BoxError(BoxError.EXTERNAL_ERROR, `Error listing objects. Message: ${listError.message} HTTP Code: ${listError.code}`); const delParams = { @@ -550,7 +524,7 @@ async function testConfig(apiConfig) { Key: path.join(apiConfig.prefix, 'snapshot/cloudron-testfile') }; - const [delError] = await safe(s3.deleteObject(delParams).promise()); + const [delError] = await safe(s3.deleteObject(delParams)); if (delError) throw new BoxError(BoxError.EXTERNAL_ERROR, `Error del object cloudron-testfile. Message: ${delError.message} HTTP Code: ${delError.code}`); } diff --git a/src/test/storage-test.js b/src/test/storage-test.js index 7c66b1512..aad9710aa 100644 --- a/src/test/storage-test.js +++ b/src/test/storage-test.js @@ -14,7 +14,6 @@ const backups = require('../backups.js'), filesystem = require('../storage/filesystem.js'), fs = require('fs'), gcs = require('../storage/gcs.js'), - MockS3 = require('mock-aws-s3'), noop = require('../storage/noop.js'), os = require('os'), path = require('path'), @@ -183,13 +182,13 @@ describe('Storage', function () { }); it('can remove empty dir', async function () { - await noop.remove(gBackupConfig, 'sourceDir', () => {}); + await noop.remove(gBackupConfig, 'sourceDir'); }); }); describe('s3', function () { - let gS3Folder; - const gBackupConfig = { + const basePath = path.join(os.tmpdir(), 's3-backup-test-buckets'); + const backupConfig = { provider: 's3', key: 'key', prefix: 'unit.test', @@ -199,69 +198,118 @@ describe('Storage', function () { region: 'eu-central-1', format: 'tgz' }; + const bucketPath = path.join(basePath, backupConfig.bucket); + + class S3MockUpload { + constructor(args) { // { client: s3, params, partSize, queueSize: 3, leavePartsOnError: false } + console.log(basePath, args.params.Bucket, args.params.Key); + const destFilePath = path.join(basePath, args.params.Bucket, args.params.Key); + fs.mkdirSync(path.dirname(destFilePath), { recursive: true }); + this.pipeline = stream.pipeline(args.params.Body, fs.createWriteStream(destFilePath)); + console.log(destFilePath); + } + + on() {} + + async done() { + await this.pipeline; + } + } + + class S3Mock { + constructor(cfg) { + expect(cfg.credentials).to.eql({ // retryDelayOptions is a function + accessKeyId: backupConfig.accessKeyId, + secretAccessKey: backupConfig.secretAccessKey + }); + expect(cfg.region).to.be(backupConfig.region); + } + + async listObjects(params) { + expect(params.Bucket).to.be(backupConfig.bucket); + return { + Contents: [{ + Key: 'uploadtest/test.txt', + Size: 23 + }, { + Key: 'uploadtest/C++.gitignore', + Size: 23 + }] + }; + } + + async copyObject(params) { + console.log(path.join(basePath, params.CopySource), path.join(bucketPath, params.Key)); + await fs.promises.mkdir(path.dirname(path.join(bucketPath, params.Key)), { recursive: true }); + await fs.promises.copyFile(path.join(basePath, params.CopySource.replace(/%2B/g, '+')), path.join(bucketPath, params.Key)); // CopySource already has the bucket path! + } + + async deleteObjects(params) { + expect(params.Bucket).to.be(backupConfig.bucket); + params.Delete.Objects.forEach(o => fs.rmSync(path.join(bucketPath, o.Key))); + } + } before(function () { - MockS3.config.basePath = path.join(os.tmpdir(), 's3-backup-test-buckets/'); - fs.rmSync(MockS3.config.basePath, { recursive: true, force: true }); - gS3Folder = path.join(MockS3.config.basePath, gBackupConfig.bucket); - - s3._mockInject(MockS3); + fs.rmSync(basePath, { recursive: true, force: true }); + globalThis.S3Mock = S3Mock; + globalThis.S3MockUpload = S3MockUpload; }); after(function () { - s3._mockRestore(); - fs.rmSync(MockS3.config.basePath, { recursive: true, force: true }); + // fs.rmSync(basePath, { recursive: true, force: true }); + delete globalThis.S3Mock; + delete globalThis.S3MockUpload; }); it('can upload', async function () { const sourceFile = path.join(__dirname, 'storage/data/test.txt'); const sourceStream = fs.createReadStream(sourceFile); const destKey = 'uploadtest/test.txt'; - const uploader = await s3.upload(gBackupConfig, destKey); + const uploader = await s3.upload(backupConfig, destKey); await stream.pipeline(sourceStream, uploader.stream); await uploader.finish(); - expect(fs.existsSync(path.join(gS3Folder, destKey))).to.be(true); - expect(fs.statSync(path.join(gS3Folder, destKey)).size).to.be(fs.statSync(sourceFile).size); + expect(fs.existsSync(path.join(bucketPath, destKey))).to.be(true); + expect(fs.statSync(path.join(bucketPath, destKey)).size).to.be(fs.statSync(sourceFile).size); }); it('can download file', async function () { const sourceKey = 'uploadtest/test.txt'; - const [error, stream] = await safe(s3.download(gBackupConfig, sourceKey)); + const [error, outstream] = await safe(s3.download(backupConfig, sourceKey)); expect(error).to.be(null); - expect(stream).to.be.an('object'); + expect(outstream).to.be.an('object'); }); it('list dir lists contents of source dir', async function () { let allFiles = [ ], marker = null; while (true) { - const result = await s3.listDir(gBackupConfig, '', 1, marker); + const result = await s3.listDir(backupConfig, '', 1, marker); allFiles = allFiles.concat(result.entries); if (!result.marker) break; marker = result.marker; } - expect(allFiles.map(function (f) { return f.fullPath; }).sort()).to.eql([ 'uploadtest/test.txt' ]); + expect(allFiles.map(function (f) { return f.fullPath; })).to.contain('uploadtest/test.txt'); }); it('can copy', async function () { - fs.writeFileSync(path.join(gS3Folder, 'uploadtest/C++.gitignore'), 'special', 'utf8'); + fs.writeFileSync(path.join(bucketPath, 'uploadtest/C++.gitignore'), 'special', 'utf8'); - const sourceKey = 'uploadtest'; - - await s3.copy(gBackupConfig, sourceKey, 'uploadtest-copy', () => {}); + await s3.copy(backupConfig, 'uploadtest', 'uploadtest-copy', () => {}); const sourceFile = path.join(__dirname, 'storage/data/test.txt'); - expect(fs.statSync(path.join(gS3Folder, 'uploadtest-copy/test.txt')).size).to.be(fs.statSync(sourceFile).size); - expect(fs.statSync(path.join(gS3Folder, 'uploadtest-copy/C++.gitignore')).size).to.be(7); + expect(fs.statSync(path.join(bucketPath, 'uploadtest-copy/test.txt')).size).to.be(fs.statSync(sourceFile).size); + expect(fs.statSync(path.join(bucketPath, 'uploadtest-copy/C++.gitignore')).size).to.be(7); }); it('can remove file', async function () { - await s3.remove(gBackupConfig, 'uploadtest-copy/test.txt'); - expect(fs.existsSync(path.join(gS3Folder, 'uploadtest-copy/test.txt'))).to.be(false); + await s3.remove(backupConfig, 'uploadtest/test.txt'); + expect(fs.existsSync(path.join(bucketPath, 'uploadtest/test.txt'))).to.be(false); }); - it('can remove non-existent dir', async function () { - await noop.remove(gBackupConfig, 'blah', () => {}); + it('cannot remove non-existent file', async function () { + const [error] = await safe(s3.remove(backupConfig, 'blah')); + expect(error).to.be.ok(); }); }); @@ -271,85 +319,91 @@ describe('Storage', function () { key: '', prefix: 'unit.test', bucket: 'cloudron-storage-test', - projectId: '', + projectId: 'some-project', credentials: { - client_email: '', - private_key: '' + client_email: 'some-client', + private_key: 'some-key' } }; + const GCSMockBasePath = path.join(os.tmpdir(), 'gcs-backup-test-buckets/'); + class GCSMockBucket { + constructor(name) { + expect(name).to.be(gBackupConfig.bucket); + } + file(filename) { + function ensurePathWritable(filename) { + filename = GCSMockBasePath + filename; + fs.mkdirSync(path.dirname(filename), { recursive: true }); + return filename; + } + + return { + name: filename, + createReadStream: function() { + return fs.createReadStream(ensurePathWritable(filename)) + .on('error', function(e){ + console.log('error createReadStream: '+filename); + if (e.code == 'ENOENT') { e.code = 404; } + this.emit('error', e); + }); + }, + createWriteStream: function() { + return fs.createWriteStream(ensurePathWritable(filename)); + }, + delete: async function() { + await fs.promises.unlink(ensurePathWritable(filename)); + }, + copy: function(dst, cb) { + function notFoundHandler(e) { + if (e && e.code == 'ENOENT') { e.code = 404; return cb(e); } + cb(); + } + + return fs.createReadStream(ensurePathWritable(filename)) + .on('end', cb) + .on('error', notFoundHandler) + .pipe(fs.createWriteStream(ensurePathWritable(dst))) + .on('end', cb) + .on('error', notFoundHandler); + } + }; + } + async getFiles(q) { + const target = path.join(GCSMockBasePath, q.prefix); + const files = execSync(`find ${target} -type f`, { encoding: 'utf8' }).trim().split('\n'); + const pageToken = q.pageToken || 0; + + const chunkedFiles = chunk(files, q.maxResults); + if (q.pageToken >= chunkedFiles.length) return [[], null]; + + const gFiles = chunkedFiles[pageToken].map(f => { + return this.file(path.relative(GCSMockBasePath, f)); + }); + + q.pageToken = pageToken + 1; + return [ gFiles, q.pageToken < chunkedFiles.length ? q : null ]; + } + }; + + class GCSMock { + constructor(config) { + expect(config.projectId).to.be(gBackupConfig.projectId); + expect(config.credentials.private_key).to.be(gBackupConfig.credentials.private_key); + } + + bucket(name) { + return new GCSMockBucket(name); + } + } before(function () { - const mockGCS = function() { - return { - bucket: function() { - const file = function (filename) { - function ensurePathWritable(filename) { - filename = GCSMockBasePath + filename; - fs.mkdirSync(path.dirname(filename), { recursive: true }); - return filename; - } - - return { - name: filename, - createReadStream: function() { - return fs.createReadStream(ensurePathWritable(filename)) - .on('error', function(e){ - console.log('error createReadStream: '+filename); - if (e.code == 'ENOENT') { e.code = 404; } - this.emit('error', e); - }); - }, - createWriteStream: function() { - return fs.createWriteStream(ensurePathWritable(filename)); - }, - delete: async function() { - await fs.promises.unlink(ensurePathWritable(filename)); - }, - copy: function(dst, cb) { - function notFoundHandler(e) { - if (e && e.code == 'ENOENT') { e.code = 404; return cb(e); } - cb(); - } - - return fs.createReadStream(ensurePathWritable(filename)) - .on('end', cb) - .on('error', notFoundHandler) - .pipe(fs.createWriteStream(ensurePathWritable(dst))) - .on('end', cb) - .on('error', notFoundHandler); - } - }; - }; - - return { - file, - - getFiles: async function(q) { - const target = path.join(GCSMockBasePath, q.prefix); - const files = execSync(`find ${target} -type f`, { encoding: 'utf8' }).trim().split('\n'); - const pageToken = q.pageToken || 0; - - const chunkedFiles = chunk(files, q.maxResults); - if (q.pageToken >= chunkedFiles.length) return [[], null]; - - const gFiles = chunkedFiles[pageToken].map(function(f) { - return file(path.relative(GCSMockBasePath, f)); //convert to gcs - }); - - q.pageToken = pageToken + 1; - return [ gFiles, q.pageToken < chunkedFiles.length ? q : null ]; - } - }; - }}; - }; - gcs._mockInject(mockGCS); + globalThis.GCSMock = GCSMock; }); - after(function (done) { - gcs._mockRestore(); + after(function () { fs.rmSync(GCSMockBasePath, { recursive: true, force: true }); - done(); + delete globalThis.GCSMock; }); it('can backup', async function () {