cloudron-box/src/storage/gcs.js

'use strict';

exports = module.exports = {
    upload: upload,
    download: download,
    downloadDir: downloadDir,
    copy: copy,

    remove: remove,
    removeDir: removeDir,

    backupDone: backupDone,

    testConfig: testConfig,

    // Used to mock GCS
    _mockInject: mockInject,
    _mockRestore: mockRestore
};

var assert = require('assert'),
    GCS = require('@google-cloud/storage'),
    BackupsError = require('../backups.js').BackupsError,
    chunk = require('lodash.chunk'),
    debug = require('debug')('box:storage/gcs'),
    once = require('once'),
    PassThrough = require('stream').PassThrough,
    EventEmitter = require('events'),
    mkdirp = require('mkdirp'),
    fs = require('fs'),
    path = require('path'),
    async = require('async');

// test only
var originalGCS;
function mockInject(mock) {
    originalGCS = GCS;
    GCS = mock;
}

function mockRestore() {
    GCS = originalGCS;
}

// internal only
function getBackupCredentials(backupConfig) {
    assert.strictEqual(typeof backupConfig, 'object');

    var config = {
        provider: backupConfig.provider,
        projectId: backupConfig.projectId,
        keyFilename: backupConfig.keyFilename,
    };

    if (backupConfig.credentials) {
        config.credentials = {
            client_email: backupConfig.credentials.client_email,
            private_key: backupConfig.credentials.private_key
        };
    }
    return config;
}
function getBucket(apiConfig) {
    var credentials = getBackupCredentials(apiConfig);
    return GCS(credentials).bucket(apiConfig.bucket);
}

// storage api


function upload(apiConfig, backupFilePath, sourceStream, callback) {
    assert.strictEqual(typeof apiConfig, 'object');
    assert.strictEqual(typeof backupFilePath, 'string');
    assert.strictEqual(typeof sourceStream, 'object');
    assert.strictEqual(typeof callback, 'function');

    function done(error) {
        if (error) {
            debug('[%s] upload: gcp upload error.', backupFilePath, error);
            return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, 'Error uploading ${backupFilePath}. Message: ${error.message} HTTP Code: ${error.code}'));
        }

        callback(null);
    }

    return sourceStream.pipe(
        getBucket(apiConfig)
            .file(backupFilePath)
            .createWriteStream({resumable: false})
                .on('finish', done)
                .on('error', function(e){
                    if (e) done(e);
                })
    );
}

function download(apiConfig, backupFilePath, callback) {
    assert.strictEqual(typeof apiConfig, 'object');
    assert.strictEqual(typeof backupFilePath, 'string');
    assert.strictEqual(typeof callback, 'function');

    var file = getBucket(apiConfig).file(backupFilePath);

    var ps = new PassThrough();
    var readStream = file.createReadStream()
        .on('error', function(error){
            if (error && error.code == 404){
                ps.emit('error', new BackupsError(BackupsError.NOT_FOUND));
            } else {
                debug('[%s] download: gcp stream error.', backupFilePath, error);
                ps.emit('error', new BackupsError(BackupsError.EXTERNAL_ERROR, error));
            }
        })
    ;
    readStream.pipe(ps);

    callback(null, ps);
}

function listDir(apiConfig, backupFilePath, batchSize, iteratorCallback, callback){
    var bucket = getBucket(apiConfig);

    var query = {prefix: backupFilePath, maxResults: batchSize, autoPaginate: false};

    async.forever(function listAndDownload(foreverCallback) {
        bucket.getFiles(query, function (error, files, nextQuery) {
            if (error) {
                debug('remove: Failed to list %s. Not fatal.', error);
                return foreverCallback(error);
            }

            var arr = batchSize === 1 ? files : chunk(files, batchSize);
            if (arr.length === 0) return foreverCallback(new Error('Done'));

            debug('emitting '+arr.length+' files found');
            iteratorCallback(arr, function (error) {
                if (error) return foreverCallback(error);

                if (arr.length < batchSize) return foreverCallback(new Error('Done'));

                query = nextQuery;
                foreverCallback();
            });
        });
    }, function (error) {
        if (error.message === 'Done') return callback(null);

        callback(error);
    });
}

function downloadDir(apiConfig, backupFilePath, destDir) {
    assert.strictEqual(typeof apiConfig, 'object');
    assert.strictEqual(typeof backupFilePath, 'string');
    assert.strictEqual(typeof destDir, 'string');

    var events = new EventEmitter();
    var total = 0;

    function downloadFile(file, iteratorCallback) {
        var relativePath = path.relative(backupFilePath, file.name);

        events.emit('progress', 'Downloading ${relativePath}');

        mkdirp(path.dirname(path.join(destDir, relativePath)), function (error) {
            if (error) return iteratorCallback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message));

            download(apiConfig, file.name, function (error, sourceStream) {
                if (error) return iteratorCallback(error);

                var destStream = fs.createWriteStream(path.join(destDir, relativePath));

                destStream.on('open', function () {
                    sourceStream.pipe(destStream);
                });

                destStream.on('error', function (error) {
                    return iteratorCallback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message));
                });

                destStream.on('finish', iteratorCallback);
            });
        });
    }

    const concurrency = 10, batchSize = 1;

    listDir(apiConfig, backupFilePath, batchSize, function (objects, done) {
        total += objects.length;
        async.eachLimit(objects, concurrency, downloadFile, done);
    }, function (error) {
        events.emit('progress', 'Downloaded ${total} files');

        events.emit('done', error);
    });

    return events;
}

function copy(apiConfig, oldFilePath, newFilePath) {
    assert.strictEqual(typeof apiConfig, 'object');
    assert.strictEqual(typeof oldFilePath, 'string');
    assert.strictEqual(typeof newFilePath, 'string');

    var events = new EventEmitter(), retryCount = 0;

    function copyFile(file, iteratorCallback){

        var relativePath = path.relative(oldFilePath, file.name);

        file.copy(path.join(newFilePath, relativePath), function(error, newFile, apiResponse){
            if (error && error.code == 404) return iteratorCallback(new BackupsError(BackupsError.NOT_FOUND, 'Old backup not found'));
            if (error) {
                debug('copyBackup: gcs copy error', error);
                return iteratorCallback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message));
            }
            iteratorCallback(null);
        });

        events.emit('progress', 'Copying (multipart) ${relativePath}');
    }

    const batchSize = 1;
    var total = 0, concurrency = 4;

    listDir(apiConfig, oldFilePath, batchSize, function (files, done) {
        total += files.length;

        if (retryCount === 0) concurrency = Math.min(concurrency + 1, 10); else concurrency = Math.max(concurrency - 1, 5);
        events.emit('progress', '${retryCount} errors. concurrency set to ${concurrency}');
        retryCount = 0;

        async.eachLimit(files, concurrency, copyFile, done);
    }, function (error) {
        events.emit('progress', 'Copied ${total} files');

        events.emit('done', error);
    });

    return events;
}

function remove(apiConfig, filename, callback) {
    assert.strictEqual(typeof apiConfig, 'object');
    assert.strictEqual(typeof filename, 'string');
    assert.strictEqual(typeof callback, 'function');

    getBucket(apiConfig)
        .file(filename)
        .delete(function(e){
            if (e) debug('removeBackups: Unable to remove %s (%s). Not fatal.', filename, e.message);
            else debug('removeBackups: Deleted: %s', filename);
            callback(null);
        });
}

function removeDir(apiConfig, pathPrefix) {
    assert.strictEqual(typeof apiConfig, 'object');
    assert.strictEqual(typeof pathPrefix, 'string');

    var events = new EventEmitter(), retryCount = 0;

    const batchSize = 1;
    var total = 0, concurrency = 4;

    listDir(apiConfig, oldFilePath, batchSize, function (files, done) {
        total += files.length;

        if (retryCount === 0) concurrency = Math.min(concurrency + 1, 10); else concurrency = Math.max(concurrency - 1, 5);
        events.emit('progress', '${retryCount} errors. concurrency set to ${concurrency}');
        retryCount = 0;

        async.eachLimit(files, concurrency, remove.bind(null, apiConfig), done);
    }, function (error) {
        events.emit('progress', 'Deleted ${total} files');

        events.emit('done', error);
    });

    return events;
}

function testConfig(apiConfig, callback) {
    assert.strictEqual(typeof apiConfig, 'object');
    assert.strictEqual(typeof callback, 'function');

    if (typeof apiConfig.projectId !== 'string') return callback(new BackupsError(BackupsError.BAD_FIELD, 'projectId must be a string'));
    if (typeof apiConfig.keyFilename !== 'string') {
        if (typeof apiConfig.credentials !== 'object') return callback(new BackupsError(BackupsError.BAD_FIELD, 'credentials must be an object'));
        if (typeof apiConfig.credentials.client_email !== 'string') return callback(new BackupsError(BackupsError.BAD_FIELD, 'credentials.client_email must be a string'));
        if (typeof apiConfig.credentials.private_key !== 'string') return callback(new BackupsError(BackupsError.BAD_FIELD, 'credentials.private_key must be a string'));
    }

    if (typeof apiConfig.bucket !== 'string') return callback(new BackupsError(BackupsError.BAD_FIELD, 'bucket must be a string'));
    if (typeof apiConfig.prefix !== 'string') return callback(new BackupsError(BackupsError.BAD_FIELD, 'prefix must be a string'));

    // attempt to upload and delete a file with new credentials
    var bucket = getBucket(apiConfig);
    var testFile = bucket.file(path.join(apiConfig.prefix, 'cloudron-testfile'));
    var uploadStream = testFile.createWriteStream({resumable: false});

    var testfileStream = new PassThrough();
    testfileStream.write("testfilecontents");
    testfileStream.end();

    testfileStream
        .on('error', function(error){
            debug('failed uploading cloudron-testfile', error);
            return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message));
        })
        .on('end', function(){
            debug('uploaded cloudron-testfile');
            testFile.delete(function(error){
                if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message));
                debug('deleted cloudron-testfile');
                callback();
            });
        })
    .pipe(uploadStream);

}

function backupDone(backupId, appBackupIds, callback) {
    assert.strictEqual(typeof backupId, 'string');
    assert(Array.isArray(appBackupIds));
    assert.strictEqual(typeof callback, 'function');

    callback();
}