301 lines
10 KiB
JavaScript
301 lines
10 KiB
JavaScript
'use strict';
|
|
|
|
exports = module.exports = {
|
|
getRootPath,
|
|
checkPreconditions,
|
|
|
|
upload,
|
|
exists,
|
|
download,
|
|
copy,
|
|
|
|
listDir,
|
|
|
|
remove,
|
|
removeDir,
|
|
|
|
remount,
|
|
|
|
testConfig,
|
|
removePrivateFields,
|
|
injectPrivateFields,
|
|
|
|
// Used to mock GCS
|
|
_mockInject: mockInject,
|
|
_mockRestore: mockRestore
|
|
};
|
|
|
|
const assert = require('assert'),
|
|
async = require('async'),
|
|
BoxError = require('../boxerror.js'),
|
|
constants = require('../constants.js'),
|
|
DataLayout = require('../datalayout.js'),
|
|
debug = require('debug')('box:storage/gcs'),
|
|
PassThrough = require('stream').PassThrough,
|
|
path = require('path'),
|
|
safe = require('safetydance'),
|
|
util = require('util');
|
|
|
|
let GCS = require('@google-cloud/storage').Storage;
|
|
|
|
// test only
|
|
let originalGCS;
|
|
function mockInject(mock) {
|
|
originalGCS = GCS;
|
|
GCS = mock;
|
|
}
|
|
|
|
function mockRestore() {
|
|
GCS = originalGCS;
|
|
}
|
|
|
|
// internal only
|
|
function getBucket(apiConfig) {
|
|
assert.strictEqual(typeof apiConfig, 'object');
|
|
|
|
const gcsConfig = {
|
|
projectId: apiConfig.projectId,
|
|
|
|
credentials: {
|
|
client_email: apiConfig.credentials.client_email,
|
|
private_key: apiConfig.credentials.private_key
|
|
}
|
|
};
|
|
|
|
return new GCS(gcsConfig).bucket(apiConfig.bucket);
|
|
}
|
|
|
|
// storage api
|
|
function getRootPath(apiConfig) {
|
|
assert.strictEqual(typeof apiConfig, 'object');
|
|
|
|
return apiConfig.prefix;
|
|
}
|
|
|
|
async function checkPreconditions(apiConfig, dataLayout) {
|
|
assert.strictEqual(typeof apiConfig, 'object');
|
|
assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
|
|
}
|
|
|
|
function upload(apiConfig, backupFilePath, sourceStream, callback) {
|
|
assert.strictEqual(typeof apiConfig, 'object');
|
|
assert.strictEqual(typeof backupFilePath, 'string');
|
|
assert.strictEqual(typeof sourceStream, 'object');
|
|
assert.strictEqual(typeof callback, 'function');
|
|
|
|
debug(`Uploading to ${backupFilePath}`);
|
|
|
|
function done(error) {
|
|
if (error) {
|
|
debug('[%s] upload: gcp upload error.', backupFilePath, error);
|
|
return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Error uploading ${backupFilePath}. Message: ${error.message} HTTP Code: ${error.code}`));
|
|
}
|
|
|
|
callback(null);
|
|
}
|
|
|
|
const uploadStream = getBucket(apiConfig).file(backupFilePath)
|
|
.createWriteStream({resumable: false})
|
|
.on('finish', done)
|
|
.on('error', done);
|
|
|
|
sourceStream.pipe(uploadStream);
|
|
}
|
|
|
|
async function exists(apiConfig, backupFilePath) {
|
|
assert.strictEqual(typeof apiConfig, 'object');
|
|
assert.strictEqual(typeof backupFilePath, 'string');
|
|
|
|
const bucket = getBucket(apiConfig);
|
|
|
|
if (!backupFilePath.endsWith('/')) {
|
|
const file = bucket.file(backupFilePath);
|
|
const [error] = await safe(file.getMetadata());
|
|
if (error && error.code === 404) return false;
|
|
if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, error.message);
|
|
|
|
return true;
|
|
} else {
|
|
const query = {
|
|
prefix: backupFilePath,
|
|
maxResults: 1,
|
|
autoPaginate: true
|
|
};
|
|
|
|
const [error, files] = await safe(bucket.getFiles(query));
|
|
if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, error.message);
|
|
|
|
return files.length !== 0;
|
|
}
|
|
}
|
|
|
|
function download(apiConfig, backupFilePath, callback) {
|
|
assert.strictEqual(typeof apiConfig, 'object');
|
|
assert.strictEqual(typeof backupFilePath, 'string');
|
|
assert.strictEqual(typeof callback, 'function');
|
|
|
|
debug(`Download ${backupFilePath} starting`);
|
|
|
|
var file = getBucket(apiConfig).file(backupFilePath);
|
|
|
|
var ps = new PassThrough();
|
|
var readStream = file.createReadStream()
|
|
.on('error', function(error) {
|
|
if (error && error.code == 404){
|
|
ps.emit('error', new BoxError(BoxError.NOT_FOUND));
|
|
} else {
|
|
debug('[%s] download: gcp stream error.', backupFilePath, error);
|
|
ps.emit('error', new BoxError(BoxError.EXTERNAL_ERROR, error));
|
|
}
|
|
})
|
|
;
|
|
readStream.pipe(ps);
|
|
|
|
callback(null, ps);
|
|
}
|
|
|
|
function listDir(apiConfig, backupFilePath, batchSize, iteratorCallback, callback) {
|
|
var bucket = getBucket(apiConfig);
|
|
|
|
var query = { prefix: backupFilePath, autoPaginate: batchSize === -1 };
|
|
if (batchSize > 0) {
|
|
query.maxResults = batchSize;
|
|
}
|
|
|
|
let done = false;
|
|
|
|
async.whilst((testDone) => testDone(null, !done), function listAndDownload(whilstCallback) {
|
|
bucket.getFiles(query, function (error, files, nextQuery) {
|
|
if (error) return whilstCallback(error);
|
|
|
|
if (files.length === 0) { done = true; return whilstCallback(); }
|
|
|
|
const entries = files.map(function (f) { return { fullPath: f.name }; });
|
|
iteratorCallback(entries, function (error) {
|
|
if (error) return whilstCallback(error);
|
|
if (!nextQuery) { done = true; return whilstCallback(); }
|
|
|
|
query = nextQuery;
|
|
|
|
whilstCallback();
|
|
});
|
|
});
|
|
}, callback);
|
|
}
|
|
|
|
async function copy(apiConfig, oldFilePath, newFilePath, progressCallback) {
|
|
assert.strictEqual(typeof apiConfig, 'object');
|
|
assert.strictEqual(typeof oldFilePath, 'string');
|
|
assert.strictEqual(typeof newFilePath, 'string');
|
|
assert.strictEqual(typeof progressCallback, 'function');
|
|
|
|
function copyFile(entry, iteratorCallback) {
|
|
var relativePath = path.relative(oldFilePath, entry.fullPath);
|
|
|
|
getBucket(apiConfig).file(entry.fullPath).copy(path.join(newFilePath, relativePath), function(error) {
|
|
if (error) debug('copyBackup: gcs copy error', error);
|
|
|
|
if (error && error.code === 404) return iteratorCallback(new BoxError(BoxError.NOT_FOUND, 'Old backup not found'));
|
|
if (error) return iteratorCallback(new BoxError(BoxError.EXTERNAL_ERROR, error.message));
|
|
|
|
iteratorCallback(null);
|
|
});
|
|
}
|
|
|
|
const batchSize = 1000;
|
|
const concurrency = apiConfig.copyConcurrency || 10;
|
|
let total = 0;
|
|
|
|
const listDirAsync = util.promisify(listDir);
|
|
|
|
const [copyError] = await safe(listDirAsync(apiConfig, oldFilePath, batchSize, function (entries, done) {
|
|
total += entries.length;
|
|
|
|
progressCallback({ message: `Copying ${entries.length} files from ${entries[0].fullPath} to ${entries[entries.length-1].fullPath}. total: ${total}` });
|
|
|
|
async.eachLimit(entries, concurrency, copyFile, done);
|
|
}));
|
|
|
|
progressCallback({ message: `Copied ${total} files with error: ${copyError}` });
|
|
}
|
|
|
|
async function remove(apiConfig, filename) {
|
|
assert.strictEqual(typeof apiConfig, 'object');
|
|
assert.strictEqual(typeof filename, 'string');
|
|
|
|
const [error] = await safe(getBucket(apiConfig).file(filename).delete());
|
|
if (error) debug('removeBackups: Unable to remove %s (%s). Not fatal.', filename, error.message);
|
|
}
|
|
|
|
async function removeDir(apiConfig, pathPrefix, progressCallback) {
|
|
assert.strictEqual(typeof apiConfig, 'object');
|
|
assert.strictEqual(typeof pathPrefix, 'string');
|
|
assert.strictEqual(typeof progressCallback, 'function');
|
|
|
|
const batchSize = 1000, concurrency = apiConfig.deleteConcurrency || 10; // https://googleapis.dev/nodejs/storage/latest/Bucket.html#deleteFiles
|
|
let total = 0;
|
|
|
|
const listDirAsync = util.promisify(listDir);
|
|
|
|
await listDirAsync(apiConfig, pathPrefix, batchSize, function (entries, done) {
|
|
total += entries.length;
|
|
|
|
progressCallback({ message: `Removing ${entries.length} files from ${entries[0].fullPath} to ${entries[entries.length-1].fullPath}. total: ${total}` });
|
|
|
|
async.eachLimit(entries, concurrency, async (entry) => await remove(apiConfig, entry.fullPath), done);
|
|
});
|
|
progressCallback({ progress: `Deleted ${total} files` });
|
|
}
|
|
|
|
async function remount(apiConfig) {
|
|
assert.strictEqual(typeof apiConfig, 'object');
|
|
}
|
|
|
|
async function testConfig(apiConfig) {
|
|
assert.strictEqual(typeof apiConfig, 'object');
|
|
|
|
if (typeof apiConfig.projectId !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'projectId must be a string');
|
|
if (!apiConfig.credentials || typeof apiConfig.credentials !== 'object') throw new BoxError(BoxError.BAD_FIELD, 'credentials must be an object');
|
|
if (typeof apiConfig.credentials.client_email !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'credentials.client_email must be a string');
|
|
if (typeof apiConfig.credentials.private_key !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'credentials.private_key must be a string');
|
|
|
|
if (typeof apiConfig.bucket !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'bucket must be a string');
|
|
if (typeof apiConfig.prefix !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'prefix must be a string');
|
|
|
|
// attempt to upload and delete a file with new credentials
|
|
const bucket = getBucket(apiConfig);
|
|
|
|
const testFile = bucket.file(path.join(apiConfig.prefix, 'cloudron-testfile'));
|
|
|
|
const uploadStream = testFile.createWriteStream({ resumable: false });
|
|
|
|
await new Promise((resolve, reject) => {
|
|
uploadStream.on('error', function(error) {
|
|
debug('testConfig: failed uploading cloudron-testfile', error);
|
|
if (error && error.code && (error.code == 403 || error.code == 404)) {
|
|
return reject(new BoxError(BoxError.BAD_FIELD, error.message));
|
|
}
|
|
|
|
return reject(new BoxError(BoxError.EXTERNAL_ERROR, error.message));
|
|
});
|
|
uploadStream.write('testfilecontents');
|
|
uploadStream.end();
|
|
|
|
uploadStream.on('finish', resolve);
|
|
});
|
|
debug('testConfig: uploaded cloudron-testfile');
|
|
|
|
const [delError] = await safe(bucket.file(path.join(apiConfig.prefix, 'cloudron-testfile')).delete());
|
|
if (delError) throw new BoxError(BoxError.EXTERNAL_ERROR, delError.message);
|
|
debug('testConfig: deleted cloudron-testfile');
|
|
}
|
|
|
|
function removePrivateFields(apiConfig) {
|
|
apiConfig.credentials.private_key = constants.SECRET_PLACEHOLDER;
|
|
return apiConfig;
|
|
}
|
|
|
|
function injectPrivateFields(newConfig, currentConfig) {
|
|
if (newConfig.credentials.private_key === constants.SECRET_PLACEHOLDER && currentConfig.credentials) newConfig.credentials.private_key = currentConfig.credentials.private_key;
|
|
}
|