cloudron-box/src/test/storage-test.js

/* global it:false */
/* global describe:false */
/* global before:false */
/* global after:false */
/* global xit:false */

'use strict';

const backups = require('../backups.js'),
    BoxError = require('../boxerror.js'),
    common = require('./common.js'),
    execSync = require('child_process').execSync,
    expect = require('expect.js'),
    filesystem = require('../storage/filesystem.js'),
    fs = require('fs'),
    gcs = require('../storage/gcs.js'),
    noop = require('../storage/noop.js'),
    os = require('os'),
    path = require('path'),
    s3 = require('../storage/s3.js'),
    safe = require('safetydance'),
    stream = require('stream/promises');

const chunk = s3._chunk;

describe('Storage', function () {
    const { setup, cleanup } = common;

    before(setup);
    after(cleanup);

    describe('filesystem', function () {
        let gTmpFolder;

        const gBackupConfig = {
            provider: 'filesystem',
            key: 'key',
            backupFolder: null,
            format: 'tgz',
        };

        before(function (done) {
            gTmpFolder = fs.mkdtempSync(path.join(os.tmpdir(), 'filesystem-storage-test_'));

            gBackupConfig.backupFolder = path.join(gTmpFolder, 'backups/');

            done();
        });

        after(function (done) {
            fs.rmSync(gTmpFolder, { recursive: true, force: true });
            done();
        });

        it('fails to set backup storage for bad folder', async function () {
            const tmp = Object.assign({}, gBackupConfig, { backupFolder: '/root/oof' });
            const [error] = await safe(backups.setStorage(tmp));
            expect(error.reason).to.equal(BoxError.BAD_FIELD);
        });

        it('succeeds to set backup storage', async function () {
            await backups.setStorage(gBackupConfig);
            expect(fs.existsSync(path.join(gBackupConfig.backupFolder, 'snapshot'))).to.be(true); // auto-created
        });

        it('can upload', async function () {
            const sourceFile = path.join(__dirname, 'storage/data/test.txt');
            const sourceStream = fs.createReadStream(sourceFile);
            const destFile = gTmpFolder + '/uploadtest/test.txt';
            const uploader = await filesystem.upload(gBackupConfig, destFile);
            await stream.pipeline(sourceStream, uploader.stream);
            await uploader.finish();
            expect(fs.existsSync(destFile));
            expect(fs.statSync(sourceFile).size).to.be(fs.statSync(destFile).size);
        });

        it('upload waits for empty file to be created', async function () {
            const sourceFile = path.join(__dirname, 'storage/data/empty');
            const sourceStream = fs.createReadStream(sourceFile);
            const destFile = gTmpFolder + '/uploadtest/empty';
            const uploader = await filesystem.upload(gBackupConfig, destFile);
            await stream.pipeline(sourceStream, uploader.stream);
            await uploader.finish();
            expect(fs.existsSync(destFile));
            expect(fs.statSync(sourceFile).size).to.be(fs.statSync(destFile).size);
        });

        it('upload unlinks old file', async function () {
            const sourceFile = path.join(__dirname, 'storage/data/test.txt');
            const sourceStream = fs.createReadStream(sourceFile);
            const destFile = gTmpFolder + '/uploadtest/test.txt';
            const oldStat = fs.statSync(destFile);
            const uploader = await filesystem.upload(gBackupConfig, destFile);
            await stream.pipeline(sourceStream, uploader.stream);
            await uploader.finish();
            expect(fs.existsSync(destFile)).to.be(true);
            expect(fs.statSync(sourceFile).size).to.be(fs.statSync(destFile).size);
            expect(oldStat.inode).to.not.be(fs.statSync(destFile).size);
        });

        it('can download file', async function () {
            const sourceFile = gTmpFolder + '/uploadtest/test.txt';

            const [error, stream] = await safe(filesystem.download(gBackupConfig, sourceFile));
            expect(error).to.be(null);
            expect(stream).to.be.an('object');
        });

        it('download errors for missing file', async function () {
            const sourceFile = gTmpFolder + '/uploadtest/missing';

            const [error] = await safe(filesystem.download(gBackupConfig, sourceFile));
            expect(error.reason).to.be(BoxError.NOT_FOUND);
        });

        it('list dir lists the source dir', async function () {
            const sourceDir = path.join(__dirname, 'storage');

            let allFiles = [], marker = null;
            while (true) {
                const result = await filesystem.listDir(gBackupConfig, sourceDir, 1, marker);
                allFiles = allFiles.concat(result.entries);
                if (!result.marker) break;
                marker = result.marker;
            }

            const expectedFiles = execSync(`find ${sourceDir} -type f`, { encoding: 'utf8' }).trim().split('\n');
            expect(allFiles.map(function (f) { return f.fullPath; }).sort()).to.eql(expectedFiles.sort());
        });

        it('can copy', async function () {
            const sourceFile = gTmpFolder + '/uploadtest/test.txt'; // keep the test within save device
            const destFile = gTmpFolder + '/uploadtest/test-hardlink.txt';

            await filesystem.copy(gBackupConfig, sourceFile, destFile, () => {});
            expect(fs.statSync(destFile).nlink).to.be(2); // created a hardlink
        });

        it('can remove file', async function () {
            const sourceFile = gTmpFolder + '/uploadtest/test-hardlink.txt';

            await filesystem.remove(gBackupConfig, sourceFile);
            expect(fs.existsSync(sourceFile)).to.be(false);
        });

        it('can remove empty dir', async function () {
            const sourceDir = gTmpFolder + '/emptydir';
            fs.mkdirSync(sourceDir);

            await filesystem.remove(gBackupConfig, sourceDir, () => {});
            expect(fs.existsSync(sourceDir)).to.be(false);
        });
    });

    describe('noop', function () {
        const gBackupConfig = {
            provider: 'noop',
            format: 'tgz'
        };

        it('upload works', async function () {
            await noop.upload(gBackupConfig, 'file', {});
        });

        it('can download file', async function () {
            const [error] = await safe(noop.download(gBackupConfig, 'file'));
            expect(error).to.be.an(Error);
        });

        it('list dir contents of source dir', async function () {
            const result = await noop.listDir(gBackupConfig, 'sourceDir', 1000, null /* marker */);
            expect(result.marker).to.be(null);
            expect(result.entries).to.eql([]);
        });

        it('can copy', async function () {
            await noop.copy(gBackupConfig, 'sourceFile', 'destFile', () => {});
        });

        it('can remove file', async function () {
            await noop.remove(gBackupConfig, 'sourceFile');
        });

        it('can remove empty dir', async function () {
            await noop.remove(gBackupConfig, 'sourceDir');
        });
    });

    describe('s3', function () {
        const basePath = path.join(os.tmpdir(), 's3-backup-test-buckets');
        const backupConfig = {
            provider: 's3',
            key: 'key',
            prefix: 'unit.test',
            bucket: 'cloudron-storage-test',
            accessKeyId: 'testkeyid',
            secretAccessKey: 'testsecret',
            region: 'eu-central-1',
            format: 'tgz'
        };
        const bucketPath = path.join(basePath, backupConfig.bucket);

        class S3MockUpload {
            constructor(args) { // { client: s3, params, partSize, queueSize: 3, leavePartsOnError: false }
                console.log(basePath, args.params.Bucket, args.params.Key);
                const destFilePath = path.join(basePath, args.params.Bucket, args.params.Key);
                fs.mkdirSync(path.dirname(destFilePath), { recursive: true });
                this.pipeline = stream.pipeline(args.params.Body, fs.createWriteStream(destFilePath));
                console.log(destFilePath);
            }

            on() {}

            async done() {
                await this.pipeline;
            }
        }

        class S3Mock {
            constructor(cfg) {
                expect(cfg.credentials).to.eql({ // retryDelayOptions is a function
                    accessKeyId: backupConfig.accessKeyId,
                    secretAccessKey: backupConfig.secretAccessKey
                });
                expect(cfg.region).to.be(backupConfig.region);
            }

            async listObjectsV2(params) {
                expect(params.Bucket).to.be(backupConfig.bucket);
                return {
                    Contents: [{
                        Key: 'uploadtest/test.txt',
                        Size: 23
                    }, {
                        Key: 'uploadtest/C++.gitignore',
                        Size: 23
                    }]
                };
            }

            async copyObject(params) {
                console.log(path.join(basePath, params.CopySource), path.join(bucketPath, params.Key));
                await fs.promises.mkdir(path.dirname(path.join(bucketPath, params.Key)), { recursive: true });
                await fs.promises.copyFile(path.join(basePath, params.CopySource.replace(/%2B/g, '+')), path.join(bucketPath, params.Key)); // CopySource already has the bucket path!
            }

            async deleteObjects(params) {
                expect(params.Bucket).to.be(backupConfig.bucket);
                params.Delete.Objects.forEach(o => fs.rmSync(path.join(bucketPath, o.Key)));
            }
        }

        before(function () {
            fs.rmSync(basePath, { recursive: true, force: true });
            globalThis.S3Mock = S3Mock;
            globalThis.S3MockUpload = S3MockUpload;
        });

        after(function () {
           // fs.rmSync(basePath, { recursive: true, force: true });
            delete globalThis.S3Mock;
            delete globalThis.S3MockUpload;
        });

        it('can upload', async function () {
            const sourceFile = path.join(__dirname, 'storage/data/test.txt');
            const sourceStream = fs.createReadStream(sourceFile);
            const destKey = 'uploadtest/test.txt';
            const uploader = await s3.upload(backupConfig, destKey);
            await stream.pipeline(sourceStream, uploader.stream);
            await uploader.finish();
            expect(fs.existsSync(path.join(bucketPath, destKey))).to.be(true);
            expect(fs.statSync(path.join(bucketPath, destKey)).size).to.be(fs.statSync(sourceFile).size);
        });

        it('can download file', async function () {
            const sourceKey = 'uploadtest/test.txt';
            const [error, outstream] = await safe(s3.download(backupConfig, sourceKey));
            expect(error).to.be(null);
            expect(outstream).to.be.an('object');
        });

        it('list dir lists contents of source dir', async function () {
            let allFiles = [ ], marker = null;

            while (true) {
                const result = await s3.listDir(backupConfig, '', 1, marker);
                allFiles = allFiles.concat(result.entries);
                if (!result.marker) break;
                marker = result.marker;
            }

            expect(allFiles.map(function (f) { return f.fullPath; })).to.contain('uploadtest/test.txt');
        });

        it('can copy', async function () {
            fs.writeFileSync(path.join(bucketPath, 'uploadtest/C++.gitignore'), 'special', 'utf8');

            await s3.copy(backupConfig, 'uploadtest', 'uploadtest-copy', () => {});
            const sourceFile = path.join(__dirname, 'storage/data/test.txt');
            expect(fs.statSync(path.join(bucketPath, 'uploadtest-copy/test.txt')).size).to.be(fs.statSync(sourceFile).size);
            expect(fs.statSync(path.join(bucketPath, 'uploadtest-copy/C++.gitignore')).size).to.be(7);
        });

        it('can remove file', async function () {
            await s3.remove(backupConfig, 'uploadtest/test.txt');
            expect(fs.existsSync(path.join(bucketPath, 'uploadtest/test.txt'))).to.be(false);
        });

        it('cannot remove non-existent file', async function () {
            const [error] = await safe(s3.remove(backupConfig, 'blah'));
            expect(error).to.be.ok();
        });
    });

    describe('gcs', function () {
        const gBackupConfig = {
            provider: 'gcs',
            key: '',
            prefix: 'unit.test',
            bucket: 'cloudron-storage-test',
            projectId: 'some-project',
            credentials: {
                client_email: 'some-client',
                private_key: 'some-key'
            }
        };

        const GCSMockBasePath = path.join(os.tmpdir(), 'gcs-backup-test-buckets/');
        class GCSMockBucket {
            constructor(name) {
                expect(name).to.be(gBackupConfig.bucket);
            }
            file(filename) {
                function ensurePathWritable(filename) {
                    filename = GCSMockBasePath + filename;
                    fs.mkdirSync(path.dirname(filename), { recursive: true });
                    return filename;
                }

                return {
                    name: filename,
                    createReadStream: function() {
                        return fs.createReadStream(ensurePathWritable(filename))
                            .on('error', function(e){
                                console.log('error createReadStream: '+filename);
                                if (e.code == 'ENOENT') { e.code = 404; }
                                this.emit('error', e);
                            });
                    },
                    createWriteStream: function() {
                        return fs.createWriteStream(ensurePathWritable(filename));
                    },
                    delete: async function() {
                        await fs.promises.unlink(ensurePathWritable(filename));
                    },
                    copy: function(dst, cb) {
                        function notFoundHandler(e) {
                            if (e && e.code == 'ENOENT') { e.code = 404; return cb(e); }
                            cb();
                        }

                        return fs.createReadStream(ensurePathWritable(filename))
                            .on('end', cb)
                            .on('error', notFoundHandler)
                            .pipe(fs.createWriteStream(ensurePathWritable(dst)))
                            .on('end', cb)
                            .on('error', notFoundHandler);
                    }
                };
            }
            async getFiles(q) {
                const target = path.join(GCSMockBasePath, q.prefix);
                const files = execSync(`find ${target} -type f`, { encoding: 'utf8' }).trim().split('\n');
                const pageToken = q.pageToken || 0;

                const chunkedFiles = chunk(files, q.maxResults);
                if (q.pageToken >= chunkedFiles.length) return [[], null];

                const gFiles = chunkedFiles[pageToken].map(f => {
                    return this.file(path.relative(GCSMockBasePath, f));
                });

                q.pageToken = pageToken + 1;
                return [ gFiles, q.pageToken < chunkedFiles.length ? q : null ];
            }
        };

        class GCSMock {
            constructor(config) {
                expect(config.projectId).to.be(gBackupConfig.projectId);
                expect(config.credentials.private_key).to.be(gBackupConfig.credentials.private_key);
            }

            bucket(name) {
                return new GCSMockBucket(name);
            }
        }

        before(function () {
            globalThis.GCSMock = GCSMock;
        });

        after(function () {
            fs.rmSync(GCSMockBasePath, { recursive: true, force: true });
            delete globalThis.GCSMock;
        });

        it('can backup', async function () {
            const sourceFile = path.join(__dirname, 'storage/data/test.txt');
            const sourceStream = fs.createReadStream(sourceFile);
            const destKey = 'uploadtest/test.txt';
            const uploader = await gcs.upload(gBackupConfig, destKey);
            await stream.pipeline(sourceStream, uploader.stream);
            await uploader.finish();
        });

        it('can download file', async function () {
            const sourceKey = 'uploadtest/test.txt';
            const [error, stream] = await safe(gcs.download(gBackupConfig, sourceKey));
            expect(error).to.be(null);
            expect(stream).to.be.an('object');
        });

        it('list dir lists contents of source dir', async function () {
            let allFiles = [ ], marker = null;

            while (true) {
                const result = await gcs.listDir(gBackupConfig, '', 1, marker);
                allFiles = allFiles.concat(result.entries);
                if (!result.marker) break;
                marker = result.marker;
            }

            expect(allFiles.map(function (f) { return f.fullPath; }).sort()).to.eql([ 'uploadtest/test.txt' ]);
        });

        xit('can copy', function (done) {
            fs.writeFileSync(path.join(GCSMockBasePath, 'uploadtest/C++.gitignore'), 'special', 'utf8');

            const sourceKey = 'uploadtest';

            const events = gcs.copy(gBackupConfig, sourceKey, 'uploadtest-copy');
            events.on('done', function (error) {
                const sourceFile = path.join(__dirname, 'storage/data/test.txt');
                expect(error).to.be(null);
                expect(fs.statSync(path.join(GCSMockBasePath, 'uploadtest-copy/test.txt')).size).to.be(fs.statSync(sourceFile).size);

                expect(fs.statSync(path.join(GCSMockBasePath, 'uploadtest-copy/C++.gitignore')).size).to.be(7);

                done();
            });
        });

        it('can remove file', async function () {
            await gcs.remove(gBackupConfig, 'uploadtest-copy/test.txt');
            expect(fs.existsSync(path.join(GCSMockBasePath, 'uploadtest-copy/test.txt'))).to.be(false);
        });

        it('can remove non-existent dir', async function () {
            await gcs.remove(gBackupConfig, 'blah', () => {});
        });
    });
});