'use strict'; exports = module.exports = { upload: upload, download: download, copy: copy, removeMany: removeMany, backupDone: backupDone, testConfig: testConfig, // Used to mock AWS _mockInject: mockInject, _mockRestore: mockRestore }; var assert = require('assert'), AWS = require('aws-sdk'), BackupsError = require('../backups.js').BackupsError, debug = require('debug')('box:storage/s3'), PassThrough = require('stream').PassThrough, path = require('path'), S3BlockReadStream = require('s3-block-read-stream'); // test only var originalAWS; function mockInject(mock) { originalAWS = AWS; AWS = mock; } function mockRestore() { AWS = originalAWS; } // internal only function getBackupCredentials(apiConfig, callback) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof callback, 'function'); assert(apiConfig.accessKeyId && apiConfig.secretAccessKey); var credentials = { signatureVersion: apiConfig.signatureVersion || 'v4', s3ForcePathStyle: true, accessKeyId: apiConfig.accessKeyId, secretAccessKey: apiConfig.secretAccessKey, region: apiConfig.region || 'us-east-1' }; if (apiConfig.endpoint) credentials.endpoint = apiConfig.endpoint; callback(null, credentials); } // storage api function upload(apiConfig, backupFilePath, sourceStream, callback) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof backupFilePath, 'string'); assert.strictEqual(typeof sourceStream, 'object'); assert.strictEqual(typeof callback, 'function'); debug('upload: %s', backupFilePath); getBackupCredentials(apiConfig, function (error, credentials) { if (error) return callback(error); var params = { Bucket: apiConfig.bucket, Key: backupFilePath, Body: sourceStream }; var s3 = new AWS.S3(credentials); // s3.upload automatically does a multi-part upload. we set queueSize to 1 to reduce memory usage s3.upload(params, { partSize: 10 * 1024 * 1024, queueSize: 1 }, function (error) { if (error) { debug('[%s] upload: s3 upload error.', backupFilePath, error); return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message)); } callback(null); }); }); } function download(apiConfig, backupFilePath, callback) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof backupFilePath, 'string'); assert.strictEqual(typeof callback, 'function'); debug('download: %s', backupFilePath); getBackupCredentials(apiConfig, function (error, credentials) { if (error) return callback(error); var params = { Bucket: apiConfig.bucket, Key: backupFilePath }; var s3 = new AWS.S3(credentials); var ps = new PassThrough(); var multipartDownload = new S3BlockReadStream(s3, params, { blockSize: 64 * 1024 * 1024, logCallback: debug }); multipartDownload.on('error', function (error) { // TODO ENOENT for the mock, fix upstream! if (error.code === 'NoSuchKey' || error.code === 'ENOENT') { ps.emit('error', new BackupsError(BackupsError.NOT_FOUND)); } else { debug('[%s] download: s3 stream error.', backupFilePath, error); ps.emit('error', new BackupsError(BackupsError.EXTERNAL_ERROR, error.message)); } }); multipartDownload.pipe(ps); callback(null, ps); }); } function copy(apiConfig, oldFilePath, newFilePath, callback) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof oldFilePath, 'string'); assert.strictEqual(typeof newFilePath, 'string'); assert.strictEqual(typeof callback, 'function'); getBackupCredentials(apiConfig, function (error, credentials) { if (error) return callback(error); var params = { Bucket: apiConfig.bucket, Key: newFilePath, CopySource: path.join(apiConfig.bucket, oldFilePath) }; var s3 = new AWS.S3(credentials); s3.copyObject(params, function (error) { if (error && error.code === 'NoSuchKey') return callback(new BackupsError(BackupsError.NOT_FOUND, 'Old backup not found')); if (error) { debug('copy: s3 copy error.', error); return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message)); } callback(null); }); }); } function removeMany(apiConfig, filePaths, callback) { assert.strictEqual(typeof apiConfig, 'object'); assert(Array.isArray(filePaths)); assert.strictEqual(typeof callback, 'function'); getBackupCredentials(apiConfig, function (error, credentials) { if (error) return callback(error); var params = { Bucket: apiConfig.bucket, Delete: { Objects: [ ] // { Key } } }; filePaths.forEach(function (filePath) { params.Delete.Objects.push({ Key: filePath }); }); var s3 = new AWS.S3(credentials); s3.deleteObjects(params, function (error, data) { if (error) debug('removeMany: Unable to remove %s. Not fatal.', params.Key, error); else debug('removeMany: Deleted: %j Errors: %j', data.Deleted, data.Errors); callback(null); }); }); } function testConfig(apiConfig, callback) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof callback, 'function'); if (typeof apiConfig.accessKeyId !== 'string') return callback(new BackupsError(BackupsError.BAD_FIELD, 'accessKeyId must be a string')); if (typeof apiConfig.secretAccessKey !== 'string') return callback(new BackupsError(BackupsError.BAD_FIELD, 'secretAccessKey must be a string')); if (typeof apiConfig.bucket !== 'string') return callback(new BackupsError(BackupsError.BAD_FIELD, 'bucket must be a string')); if (typeof apiConfig.prefix !== 'string') return callback(new BackupsError(BackupsError.BAD_FIELD, 'prefix must be a string')); if ('signatureVersion' in apiConfig && typeof apiConfig.prefix !== 'string') return callback(new BackupsError(BackupsError.BAD_FIELD, 'signatureVersion must be a string')); if ('endpoint' in apiConfig && typeof apiConfig.prefix !== 'string') return callback(new BackupsError(BackupsError.BAD_FIELD, 'endpoint must be a string')); // attempt to upload and delete a file with new credentials getBackupCredentials(apiConfig, function (error, credentials) { if (error) return callback(error); var params = { Bucket: apiConfig.bucket, Key: path.join(apiConfig.prefix, 'cloudron-testfile'), Body: 'testcontent' }; var s3 = new AWS.S3(credentials); s3.putObject(params, function (error) { if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message)); var params = { Bucket: apiConfig.bucket, Key: path.join(apiConfig.prefix, 'cloudron-testfile') }; s3.deleteObject(params, function (error) { if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message)); callback(); }); }); }); } function backupDone(backupId, appBackupIds, callback) { assert.strictEqual(typeof backupId, 'string'); assert(Array.isArray(appBackupIds)); assert.strictEqual(typeof callback, 'function'); callback(); }