'use strict'; exports = module.exports = { getBackupRootPath, getProviderStatus, getAvailableSize, upload, exists, download, copy, listDir, remove, removeDir, testConfig, removePrivateFields, injectPrivateFields, // Used to mock GCS _mockInject: mockInject, _mockRestore: mockRestore }; const assert = require('assert'), async = require('async'), BoxError = require('../boxerror.js'), constants = require('../constants.js'), debug = require('debug')('box:storage/gcs'), path = require('path'), safe = require('safetydance'), util = require('util'); let GCS = require('@google-cloud/storage').Storage; // test only let originalGCS; function mockInject(mock) { originalGCS = GCS; GCS = mock; } function mockRestore() { GCS = originalGCS; } // internal only function getBucket(apiConfig) { assert.strictEqual(typeof apiConfig, 'object'); const gcsConfig = { projectId: apiConfig.projectId, credentials: { client_email: apiConfig.credentials.client_email, private_key: apiConfig.credentials.private_key } }; return new GCS(gcsConfig).bucket(apiConfig.bucket); } // storage api function getBackupRootPath(apiConfig) { assert.strictEqual(typeof apiConfig, 'object'); return apiConfig.prefix; } async function getProviderStatus(apiConfig) { assert.strictEqual(typeof apiConfig, 'object'); return { state: 'active' }; } async function getAvailableSize(apiConfig) { assert.strictEqual(typeof apiConfig, 'object'); return Number.POSITIVE_INFINITY; } function upload(apiConfig, backupFilePath, sourceStream, callback) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof backupFilePath, 'string'); assert.strictEqual(typeof sourceStream, 'object'); assert.strictEqual(typeof callback, 'function'); debug(`Uploading to ${backupFilePath}`); function done(error) { if (error) { debug(`upload: [${backupFilePath}] gcp upload error. %o`, error); return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Error uploading ${backupFilePath}. Message: ${error.message} HTTP Code: ${error.code}`)); } callback(null); } const uploadStream = getBucket(apiConfig).file(backupFilePath) .createWriteStream({resumable: false}) .on('finish', done) .on('error', done); sourceStream.pipe(uploadStream); } async function exists(apiConfig, backupFilePath) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof backupFilePath, 'string'); const bucket = getBucket(apiConfig); if (!backupFilePath.endsWith('/')) { const file = bucket.file(backupFilePath); const [error] = await safe(file.getMetadata()); if (error && error.code === 404) return false; if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, error.message); return true; } else { const query = { prefix: backupFilePath, maxResults: 1, autoPaginate: true }; const [error, files] = await safe(bucket.getFiles(query)); if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, error.message); return files.length !== 0; } } async function download(apiConfig, backupFilePath) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof backupFilePath, 'string'); debug(`Download ${backupFilePath} starting`); const file = getBucket(apiConfig).file(backupFilePath); return file.createReadStream(); } function listDir(apiConfig, backupFilePath, batchSize, iteratorCallback, callback) { var bucket = getBucket(apiConfig); var query = { prefix: backupFilePath, autoPaginate: batchSize === -1 }; if (batchSize > 0) { query.maxResults = batchSize; } let done = false; async.whilst((testDone) => testDone(null, !done), function listAndDownload(whilstCallback) { bucket.getFiles(query, function (error, files, nextQuery) { if (error) return whilstCallback(error); if (files.length === 0) { done = true; return whilstCallback(); } const entries = files.map(function (f) { return { fullPath: f.name }; }); iteratorCallback(entries, function (error) { if (error) return whilstCallback(error); if (!nextQuery) { done = true; return whilstCallback(); } query = nextQuery; whilstCallback(); }); }); }, callback); } async function copy(apiConfig, oldFilePath, newFilePath, progressCallback) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof oldFilePath, 'string'); assert.strictEqual(typeof newFilePath, 'string'); assert.strictEqual(typeof progressCallback, 'function'); function copyFile(entry, iteratorCallback) { var relativePath = path.relative(oldFilePath, entry.fullPath); getBucket(apiConfig).file(entry.fullPath).copy(path.join(newFilePath, relativePath), function(error) { if (error) debug('copyBackup: gcs copy error. %o', error); if (error && error.code === 404) return iteratorCallback(new BoxError(BoxError.NOT_FOUND, 'Old backup not found')); if (error) return iteratorCallback(new BoxError(BoxError.EXTERNAL_ERROR, error.message)); iteratorCallback(null); }); } const batchSize = 1000; const concurrency = apiConfig.limits?.copyConcurrency || 10; let total = 0; const listDirAsync = util.promisify(listDir); const [copyError] = await safe(listDirAsync(apiConfig, oldFilePath, batchSize, function (entries, done) { total += entries.length; progressCallback({ message: `Copying ${entries.length} files from ${entries[0].fullPath} to ${entries[entries.length-1].fullPath}. total: ${total}` }); async.eachLimit(entries, concurrency, copyFile, done); })); progressCallback({ message: `Copied ${total} files with error: ${copyError}` }); } async function remove(apiConfig, filename) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof filename, 'string'); const [error] = await safe(getBucket(apiConfig).file(filename).delete()); if (error) debug('removeBackups: Unable to remove %s (%s). Not fatal.', filename, error.message); } async function removeDir(apiConfig, pathPrefix, progressCallback) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof pathPrefix, 'string'); assert.strictEqual(typeof progressCallback, 'function'); const batchSize = 1000, concurrency = apiConfig.limits?.deleteConcurrency || 10; // https://googleapis.dev/nodejs/storage/latest/Bucket.html#deleteFiles let total = 0; const listDirAsync = util.promisify(listDir); await listDirAsync(apiConfig, pathPrefix, batchSize, function (entries, done) { total += entries.length; progressCallback({ message: `Removing ${entries.length} files from ${entries[0].fullPath} to ${entries[entries.length-1].fullPath}. total: ${total}` }); async.eachLimit(entries, concurrency, async (entry) => await remove(apiConfig, entry.fullPath), done); }); progressCallback({ progress: `Deleted ${total} files` }); } async function testConfig(apiConfig) { assert.strictEqual(typeof apiConfig, 'object'); if (typeof apiConfig.projectId !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'projectId must be a string'); if (!apiConfig.credentials || typeof apiConfig.credentials !== 'object') throw new BoxError(BoxError.BAD_FIELD, 'credentials must be an object'); if (typeof apiConfig.credentials.client_email !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'credentials.client_email must be a string'); if (typeof apiConfig.credentials.private_key !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'credentials.private_key must be a string'); if (typeof apiConfig.bucket !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'bucket must be a string'); if (typeof apiConfig.prefix !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'prefix must be a string'); // attempt to upload and delete a file with new credentials const bucket = getBucket(apiConfig); const testFile = bucket.file(path.join(apiConfig.prefix, 'cloudron-testfile')); const uploadStream = testFile.createWriteStream({ resumable: false }); await new Promise((resolve, reject) => { uploadStream.on('error', function(error) { debug('testConfig: failed uploading cloudron-testfile. %o', error); if (error && error.code && (error.code == 403 || error.code == 404)) { return reject(new BoxError(BoxError.BAD_FIELD, error.message)); } return reject(new BoxError(BoxError.EXTERNAL_ERROR, error.message)); }); uploadStream.write('testfilecontents'); uploadStream.end(); uploadStream.on('finish', resolve); }); debug('testConfig: uploaded cloudron-testfile'); const [delError] = await safe(bucket.file(path.join(apiConfig.prefix, 'cloudron-testfile')).delete()); if (delError) throw new BoxError(BoxError.EXTERNAL_ERROR, delError.message); debug('testConfig: deleted cloudron-testfile'); } function removePrivateFields(apiConfig) { apiConfig.credentials.private_key = constants.SECRET_PLACEHOLDER; return apiConfig; } function injectPrivateFields(newConfig, currentConfig) { if (newConfig.credentials.private_key === constants.SECRET_PLACEHOLDER && currentConfig.credentials) newConfig.credentials.private_key = currentConfig.credentials.private_key; }