diff --git a/src/backups.js b/src/backups.js index 53cc893de..c0cb337ce 100644 --- a/src/backups.js +++ b/src/backups.js @@ -270,6 +270,20 @@ function tarExtract(inStream, destination, key, callback) { } } +function createEmptyDirs(appDataDir, callback) { + assert.strictEqual(typeof appDataDir, 'string'); + assert.strictEqual(typeof callback, 'function'); + + debugApp('createEmptyDirs: recreating empty directories'); + + var emptyDirs = safe.fs.readFileSync(path.join(appDataDir, 'emptydirs.txt'), 'utf8'); + if (!emptyDirs) return callback(new Error('emptydirs.txt was not found:' + safe.fs.error)); + + async.eachSeries(emptyDirs.trim().split('\n'), function createPath(emptyDir, iteratorDone) { + mkdirp(path.join(appDataDir, 'data', emptyDir), iteratorDone); + }, callback); +} + function download(backupId, dataDir, callback) { assert.strictEqual(typeof backupId, 'string'); assert.strictEqual(typeof dataDir, 'string'); @@ -283,11 +297,18 @@ function download(backupId, dataDir, callback) { mkdirp(getBackupFilePath(backupConfig, dataDir), function (error) { if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message)); - api(backupConfig.provider).download(backupConfig, getBackupFilePath(backupConfig, backupId), function (error, sourceStream) { - if (error) return callback(error); + if (backupConfig.format === 'tgz') { + api(backupConfig.provider).download(backupConfig, getBackupFilePath(backupConfig, backupId), function (error, sourceStream) { + if (error) return callback(error); - tarExtract(sourceStream, dataDir, backupConfig.key || null, callback); - }); + tarExtract(sourceStream, dataDir, backupConfig.key || null, callback); + }); + } else { + async.series([ + api(backupConfig.provider).downloadDir.bind(null, backupConfig, getBackupFilePath(backupConfig, backupId), dataDir), + createEmptyDirs.bind(null, dataDir) + ], callback); + } }); }); } diff --git a/src/routes/test/sysadmin-test.js b/src/routes/test/sysadmin-test.js index ef362f903..621fa9231 100644 --- a/src/routes/test/sysadmin-test.js +++ b/src/routes/test/sysadmin-test.js @@ -53,7 +53,7 @@ function setup(done) { }, function createSettings(callback) { - settings.setBackupConfig({ provider: 'caas', token: 'BACKUP_TOKEN', bucket: 'Bucket', prefix: 'Prefix' }, callback); + settings.setBackupConfig({ provider: 'caas', token: 'BACKUP_TOKEN', bucket: 'Bucket', prefix: 'Prefix', format: 'tgz' }, callback); } ], done); } diff --git a/src/storage/caas.js b/src/storage/caas.js index ae7e3d69e..989779093 100644 --- a/src/storage/caas.js +++ b/src/storage/caas.js @@ -3,6 +3,7 @@ exports = module.exports = { upload: upload, download: download, + downloadDir: downloadDir, copy: copy, remove: remove, @@ -17,6 +18,8 @@ var assert = require('assert'), BackupsError = require('../backups.js').BackupsError, config = require('../config.js'), debug = require('debug')('box:storage/caas'), + fs = require('fs'), + mkdirp = require('mkdirp'), PassThrough = require('stream').PassThrough, path = require('path'), S3BlockReadStream = require('s3-block-read-stream'), @@ -114,6 +117,54 @@ function download(apiConfig, backupFilePath, callback) { }); } +function downloadDir(apiConfig, backupFilePath, destDir, callback) { + assert.strictEqual(typeof apiConfig, 'object'); + assert.strictEqual(typeof backupFilePath, 'string'); + assert.strictEqual(typeof destDir, 'string'); + assert.strictEqual(typeof callback, 'function'); + + getBackupCredentials(apiConfig, function (error, credentials) { + if (error) return callback(error); + + var s3 = new AWS.S3(credentials); + var listParams = { + Bucket: apiConfig.bucket, + Prefix: backupFilePath + }; + + async.forever(function listAndDownload(foreverCallback) { + s3.listObjectsV2(listParams, function (error, listData) { + if (error) { + debug('remove: Failed to list %s. Not fatal.', error); + return foreverCallback(error); + } + + async.eachLimit(listData.Contents, 10, function downloadFile(content, iteratorCallback) { + var relativePath = path.relative(backupFilePath, content.Key); + mkdirp(path.dirname(path.join(destDir, relativePath)), function (error) { + if (error) return iteratorCallback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message)); + + var destStream = fs.createWriteStream(path.join(destDir, relativePath)); + destStream.on('error', function (error) { + return iteratorCallback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message)); + }); + download(apiConfig, content.Key, destStream, iteratorCallback); + }); + }, function doneCopying(error) { + if (error) return foreverCallback(error); + + if (listData.IsTruncated) return foreverCallback(); + + foreverCallback(new Error('Done')); + }); + }); + }, function (error) { + if (error.message === 'Done') return callback(); + callback(error); + }); + }); +} + function copy(apiConfig, oldFilePath, newFilePath, callback) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof oldFilePath, 'string'); diff --git a/src/storage/filesystem.js b/src/storage/filesystem.js index ac1f56370..fe6c78c94 100644 --- a/src/storage/filesystem.js +++ b/src/storage/filesystem.js @@ -3,6 +3,8 @@ exports = module.exports = { upload: upload, download: download, + downloadDir: downloadDir, + copy: copy, remove: remove, @@ -74,6 +76,21 @@ function download(apiConfig, sourceFilePath, callback) { callback(null, ps); } +function downloadDir(apiConfig, backupFilePath, destDir, callback) { + assert.strictEqual(typeof apiConfig, 'object'); + assert.strictEqual(typeof backupFilePath, 'string'); + assert.strictEqual(typeof destDir, 'string'); + assert.strictEqual(typeof callback, 'function'); + + debug('downloadDir: %s -> %s', backupFilePath, destDir); + + shell.exec('copy', '/bin/cp', [ '-r', backupFilePath + '/.', destDir ], { }, function (error) { + if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message)); + + callback(); + }); +} + function copy(apiConfig, oldFilePath, newFilePath, callback) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof oldFilePath, 'string'); diff --git a/src/storage/interface.js b/src/storage/interface.js index 78a7af2f5..d13df9ca8 100644 --- a/src/storage/interface.js +++ b/src/storage/interface.js @@ -9,6 +9,7 @@ exports = module.exports = { upload: upload, download: download, + downloadDir: downloadDir, copy: copy, remove: remove, @@ -40,6 +41,15 @@ function download(apiConfig, backupFilePath, callback) { callback(new Error('not implemented')); } +function downloadDir(apiConfig, backupFilePath, destDir, callback) { + assert.strictEqual(typeof apiConfig, 'object'); + assert.strictEqual(typeof backupFilePath, 'string'); + assert.strictEqual(typeof destDir, 'string'); + assert.strictEqual(typeof callback, 'function'); + + callback(new Error('not implemented')); +} + function copy(apiConfig, oldFilePath, newFilePath, callback) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof oldFilePath, 'string'); diff --git a/src/storage/noop.js b/src/storage/noop.js index c4d9a4b78..bd5560309 100644 --- a/src/storage/noop.js +++ b/src/storage/noop.js @@ -3,6 +3,7 @@ exports = module.exports = { upload: upload, download: download, + downloadDir: downloadDir, copy: copy, remove: remove, @@ -36,6 +37,17 @@ function download(apiConfig, backupFilePath, callback) { callback(new Error('Cannot download from noop backend')); } +function downloadDir(apiConfig, backupFilePath, destDir, callback) { + assert.strictEqual(typeof apiConfig, 'object'); + assert.strictEqual(typeof backupFilePath, 'string'); + assert.strictEqual(typeof destDir, 'string'); + assert.strictEqual(typeof callback, 'function'); + + debug('downloadDir: %s -> %s', backupFilePath, destDir); + + callback(); +} + function copy(apiConfig, oldFilePath, newFilePath, callback) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof oldFilePath, 'string'); diff --git a/src/storage/s3.js b/src/storage/s3.js index 679b4f486..b445006f4 100644 --- a/src/storage/s3.js +++ b/src/storage/s3.js @@ -3,6 +3,7 @@ exports = module.exports = { upload: upload, download: download, + downloadDir: downloadDir, copy: copy, remove: remove, @@ -21,6 +22,8 @@ var assert = require('assert'), AWS = require('aws-sdk'), BackupsError = require('../backups.js').BackupsError, debug = require('debug')('box:storage/s3'), + fs = require('fs'), + mkdirp = require('mkdirp'), PassThrough = require('stream').PassThrough, path = require('path'), S3BlockReadStream = require('s3-block-read-stream'); @@ -123,6 +126,54 @@ function download(apiConfig, backupFilePath, callback) { }); } +function downloadDir(apiConfig, backupFilePath, destDir, callback) { + assert.strictEqual(typeof apiConfig, 'object'); + assert.strictEqual(typeof backupFilePath, 'string'); + assert.strictEqual(typeof destDir, 'string'); + assert.strictEqual(typeof callback, 'function'); + + getBackupCredentials(apiConfig, function (error, credentials) { + if (error) return callback(error); + + var s3 = new AWS.S3(credentials); + var listParams = { + Bucket: apiConfig.bucket, + Prefix: backupFilePath + }; + + async.forever(function listAndDownload(foreverCallback) { + s3.listObjectsV2(listParams, function (error, listData) { + if (error) { + debug('remove: Failed to list %s. Not fatal.', error); + return foreverCallback(error); + } + + async.eachLimit(listData.Contents, 10, function downloadFile(content, iteratorCallback) { + var relativePath = path.relative(backupFilePath, content.Key); + mkdirp(path.dirname(path.join(destDir, relativePath)), function (error) { + if (error) return iteratorCallback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message)); + + var destStream = fs.createWriteStream(path.join(destDir, relativePath)); + destStream.on('error', function (error) { + return iteratorCallback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message)); + }); + download(apiConfig, content.Key, destStream, iteratorCallback); + }); + }, function doneCopying(error) { + if (error) return foreverCallback(error); + + if (listData.IsTruncated) return foreverCallback(); + + foreverCallback(new Error('Done')); + }); + }); + }, function (error) { + if (error.message === 'Done') return callback(); + callback(error); + }); + }); +} + function copy(apiConfig, oldFilePath, newFilePath, callback) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof oldFilePath, 'string'); @@ -188,7 +239,7 @@ function remove(apiConfig, pathPrefix, callback) { listParams.Marker = listData.Contents[listData.Contents.length - 1].Key; // NextMarker is returned only with delimiter - if (deleteData.IsTruncated) return iteratorCallback(); + if (listData.IsTruncated) return iteratorCallback(); iteratorCallback(new Error('Done')); }); diff --git a/src/test/storage-test.js b/src/test/storage-test.js index 7b1a8c00e..1b9903f86 100644 --- a/src/test/storage-test.js +++ b/src/test/storage-test.js @@ -101,7 +101,8 @@ describe('Storage', function () { var gBackupConfig = { provider: 'filesystem', key: 'key', - backupFolder: null + backupFolder: null, + format: 'tgz' }; before(function (done) { @@ -124,8 +125,8 @@ describe('Storage', function () { after(function (done) { cleanup(function (error) { expect(error).to.be(null); + rimraf.sync(gTmpFolder); done(); - // rimraf(gTmpFolder, done); }); }); @@ -240,7 +241,8 @@ describe('Storage', function () { bucket: 'cloudron-storage-test', accessKeyId: 'testkeyid', secretAccessKey: 'testsecret', - region: 'eu-central-1' + region: 'eu-central-1', + format: 'tgz' }; before(function (done) { @@ -269,7 +271,8 @@ describe('Storage', function () { cleanup(function (error) { expect(error).to.be(null); - rimraf(gTmpFolder, done); + rimraf.sync(gTmpFolder); + done(); }); });