cloudron-box/src/storage/gcs.js

'use strict';

exports = module.exports = {
    getBackupRootPath,
    getProviderStatus,
    getAvailableSize,

    upload,
    exists,
    download,
    copy,

    listDir,

    remove,
    removeDir,

    testConfig,
    removePrivateFields,
    injectPrivateFields,

    // Used to mock GCS
    _mockInject: mockInject,
    _mockRestore: mockRestore
};

const assert = require('assert'),
    async = require('async'),
    BoxError = require('../boxerror.js'),
    constants = require('../constants.js'),
    debug = require('debug')('box:storage/gcs'),
    path = require('path'),
    safe = require('safetydance'),
    util = require('util');

let GCS = require('@google-cloud/storage').Storage;

// test only
let originalGCS;
function mockInject(mock) {
    originalGCS = GCS;
    GCS = mock;
}

function mockRestore() {
    GCS = originalGCS;
}

// internal only
function getBucket(apiConfig) {
    assert.strictEqual(typeof apiConfig, 'object');

    const gcsConfig = {
        projectId: apiConfig.projectId,

        credentials: {
            client_email: apiConfig.credentials.client_email,
            private_key: apiConfig.credentials.private_key
        }
    };

    return new GCS(gcsConfig).bucket(apiConfig.bucket);
}

// storage api
function getBackupRootPath(apiConfig) {
    assert.strictEqual(typeof apiConfig, 'object');

    return apiConfig.prefix;
}

async function getProviderStatus(apiConfig) {
    assert.strictEqual(typeof apiConfig, 'object');

    return { state: 'active' };
}

async function getAvailableSize(apiConfig) {
    assert.strictEqual(typeof apiConfig, 'object');

    return Number.POSITIVE_INFINITY;
}

function upload(apiConfig, backupFilePath, sourceStream, callback) {
    assert.strictEqual(typeof apiConfig, 'object');
    assert.strictEqual(typeof backupFilePath, 'string');
    assert.strictEqual(typeof sourceStream, 'object');
    assert.strictEqual(typeof callback, 'function');

    debug(`Uploading to ${backupFilePath}`);

    function done(error) {
        if (error) {
            debug(`upload: [${backupFilePath}] gcp upload error. %o`, error);
            return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Error uploading ${backupFilePath}. Message: ${error.message} HTTP Code: ${error.code}`));
        }

        callback(null);
    }

    const uploadStream = getBucket(apiConfig).file(backupFilePath)
        .createWriteStream({resumable: false})
        .on('finish', done)
        .on('error', done);

    sourceStream.pipe(uploadStream);
}

async function exists(apiConfig, backupFilePath) {
    assert.strictEqual(typeof apiConfig, 'object');
    assert.strictEqual(typeof backupFilePath, 'string');

    const bucket = getBucket(apiConfig);

    if (!backupFilePath.endsWith('/')) {
        const file = bucket.file(backupFilePath);
        const [error] = await safe(file.getMetadata());
        if (error && error.code === 404) return false;
        if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, error.message);

        return true;
    } else {
        const query = {
            prefix: backupFilePath,
            maxResults: 1,
            autoPaginate: true
        };

        const [error, files] = await safe(bucket.getFiles(query));
        if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, error.message);

        return files.length !== 0;
    }
}

async function download(apiConfig, backupFilePath) {
    assert.strictEqual(typeof apiConfig, 'object');
    assert.strictEqual(typeof backupFilePath, 'string');

    debug(`Download ${backupFilePath} starting`);

    const file = getBucket(apiConfig).file(backupFilePath);
    return file.createReadStream();
}

function listDir(apiConfig, backupFilePath, batchSize, iteratorCallback, callback) {
    var bucket = getBucket(apiConfig);

    var query = { prefix: backupFilePath, autoPaginate: batchSize === -1 };
    if (batchSize > 0) {
        query.maxResults = batchSize;
    }

    let done = false;

    async.whilst((testDone) => testDone(null, !done), function listAndDownload(whilstCallback) {
        bucket.getFiles(query, function (error, files, nextQuery) {
            if (error) return whilstCallback(error);

            if (files.length === 0) { done = true; return whilstCallback(); }

            const entries = files.map(function (f) { return { fullPath: f.name }; });
            iteratorCallback(entries, function (error) {
                if (error) return whilstCallback(error);
                if (!nextQuery) { done = true; return whilstCallback(); }

                query = nextQuery;

                whilstCallback();
            });
        });
    }, callback);
}

async function copy(apiConfig, oldFilePath, newFilePath, progressCallback) {
    assert.strictEqual(typeof apiConfig, 'object');
    assert.strictEqual(typeof oldFilePath, 'string');
    assert.strictEqual(typeof newFilePath, 'string');
    assert.strictEqual(typeof progressCallback, 'function');

    function copyFile(entry, iteratorCallback) {
        var relativePath = path.relative(oldFilePath, entry.fullPath);

        getBucket(apiConfig).file(entry.fullPath).copy(path.join(newFilePath, relativePath), function(error) {
            if (error) debug('copyBackup: gcs copy error. %o', error);

            if (error && error.code === 404) return iteratorCallback(new BoxError(BoxError.NOT_FOUND, 'Old backup not found'));
            if (error) return iteratorCallback(new BoxError(BoxError.EXTERNAL_ERROR, error.message));

            iteratorCallback(null);
        });
    }

    const batchSize = 1000;
    const concurrency = apiConfig.limits?.copyConcurrency || 10;
    let total = 0;

    const listDirAsync = util.promisify(listDir);

    const [copyError] = await safe(listDirAsync(apiConfig, oldFilePath, batchSize, function (entries, done) {
        total += entries.length;

        progressCallback({ message: `Copying ${entries.length} files from ${entries[0].fullPath} to ${entries[entries.length-1].fullPath}. total: ${total}` });

        async.eachLimit(entries, concurrency, copyFile, done);
    }));

    progressCallback({ message: `Copied ${total} files with error: ${copyError}` });
}

async function remove(apiConfig, filename) {
    assert.strictEqual(typeof apiConfig, 'object');
    assert.strictEqual(typeof filename, 'string');

    const [error] = await safe(getBucket(apiConfig).file(filename).delete());
    if (error) debug('removeBackups: Unable to remove %s (%s). Not fatal.', filename, error.message);
}

async function removeDir(apiConfig, pathPrefix, progressCallback) {
    assert.strictEqual(typeof apiConfig, 'object');
    assert.strictEqual(typeof pathPrefix, 'string');
    assert.strictEqual(typeof progressCallback, 'function');

    const batchSize = 1000, concurrency = apiConfig.limits?.deleteConcurrency || 10; // https://googleapis.dev/nodejs/storage/latest/Bucket.html#deleteFiles
    let total = 0;

    const listDirAsync = util.promisify(listDir);

    await listDirAsync(apiConfig, pathPrefix, batchSize, function (entries, done) {
        total += entries.length;

        progressCallback({ message: `Removing ${entries.length} files from ${entries[0].fullPath} to ${entries[entries.length-1].fullPath}. total: ${total}` });

        async.eachLimit(entries, concurrency, async (entry) => await remove(apiConfig, entry.fullPath), done);
    });
    progressCallback({ progress: `Deleted ${total} files` });
}

async function testConfig(apiConfig) {
    assert.strictEqual(typeof apiConfig, 'object');

    if (typeof apiConfig.projectId !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'projectId must be a string');
    if (!apiConfig.credentials || typeof apiConfig.credentials !== 'object') throw new BoxError(BoxError.BAD_FIELD, 'credentials must be an object');
    if (typeof apiConfig.credentials.client_email !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'credentials.client_email must be a string');
    if (typeof apiConfig.credentials.private_key !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'credentials.private_key must be a string');

    if (typeof apiConfig.bucket !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'bucket must be a string');
    if (typeof apiConfig.prefix !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'prefix must be a string');

    // attempt to upload and delete a file with new credentials
    const bucket = getBucket(apiConfig);

    const testFile = bucket.file(path.join(apiConfig.prefix, 'cloudron-testfile'));

    const uploadStream = testFile.createWriteStream({ resumable: false });

    await new Promise((resolve, reject) => {
        uploadStream.on('error', function(error) {
            debug('testConfig: failed uploading cloudron-testfile. %o', error);
            if (error && error.code && (error.code == 403 || error.code == 404)) {
                return reject(new BoxError(BoxError.BAD_FIELD, error.message));
            }

            return reject(new BoxError(BoxError.EXTERNAL_ERROR, error.message));
        });
        uploadStream.write('testfilecontents');
        uploadStream.end();

        uploadStream.on('finish', resolve);
    });
    debug('testConfig: uploaded cloudron-testfile');

    const [delError] = await safe(bucket.file(path.join(apiConfig.prefix, 'cloudron-testfile')).delete());
    if (delError) throw new BoxError(BoxError.EXTERNAL_ERROR, delError.message);
    debug('testConfig: deleted cloudron-testfile');
}

function removePrivateFields(apiConfig) {
    apiConfig.credentials.private_key = constants.SECRET_PLACEHOLDER;
    return apiConfig;
}

function injectPrivateFields(newConfig, currentConfig) {
    if (newConfig.credentials.private_key === constants.SECRET_PLACEHOLDER && currentConfig.credentials) newConfig.credentials.private_key = currentConfig.credentials.private_key;
}