diff --git a/migrations/20171116191443-backups-add-manifestJson.js.js b/migrations/20171116191443-backups-add-manifestJson.js.js new file mode 100644 index 000000000..50cec1bd8 --- /dev/null +++ b/migrations/20171116191443-backups-add-manifestJson.js.js @@ -0,0 +1,41 @@ +'use strict'; + +var async = require('async'); + +exports.up = function(db, callback) { + async.series([ + db.runSql.bind(db, 'ALTER TABLE backups ADD COLUMN manifestJson TEXT'), + + db.runSql.bind(db, 'START TRANSACTION;'), + + // fill all the backups with restoreConfigs from current apps + function addManifests(callback) { + console.log('Importing manifests'); + + db.all('SELECT * FROM backups WHERE type="app"', function (error, backups) { + if (error) return callback(error); + + async.eachSeries(backups, function (backup, next) { + var m = backup.restoreConfigJson ? JSON.stringify(JSON.parse(backup.restoreConfigJson).manifest) : null; + + db.runSql('UPDATE backups SET manifestJson=? WHERE id=?', [ m, backup.id ], next); + }); + }); + + callback(); + }, + + db.runSql.bind(db, 'COMMIT'), + + // remove the restoreConfig + db.runSql.bind(db, 'ALTER TABLE backups DROP COLUMN restoreConfigJson') + ], callback); +}; + +exports.down = function(db, callback) { + async.series([ + db.runSql.bind(db, 'ALTER TABLE backups DROP COLUMN manifestJson'), + db.runSql.bind(db, 'ALTER TABLE backups ADD COLUMN restoreConfigJson TEXT'), + ], callback); +}; + diff --git a/migrations/schema.sql b/migrations/schema.sql index 94dd4fd01..7b8e405e3 100644 --- a/migrations/schema.sql +++ b/migrations/schema.sql @@ -111,7 +111,7 @@ CREATE TABLE IF NOT EXISTS backups( type VARCHAR(16) NOT NULL, /* 'box' or 'app' */ dependsOn TEXT, /* comma separate list of objects this backup depends on */ state VARCHAR(16) NOT NULL, - restoreConfigJson TEXT, /* JSON including the manifest of the backed up app */ + manifestJson TEXT, /* to validate if the app can be installed in this version of box */ format VARCHAR(16) DEFAULT "tgz", PRIMARY KEY (id)); diff --git a/src/apps.js b/src/apps.js index 4fb20ae88..3367f5122 100644 --- a/src/apps.js +++ b/src/apps.js @@ -761,22 +761,22 @@ function restore(appId, data, auditSource, callback) { if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error)); // for empty or null backupId, use existing manifest to mimic a reinstall - var func = data.backupId ? backups.getRestoreConfig.bind(null, data.backupId) : function (next) { return next(null, { manifest: app.manifest }); }; + var func = data.backupId ? backups.get.bind(null, data.backupId) : function (next) { return next(null, { manifest: app.manifest }); }; - func(function (error, restoreConfig) { + func(function (error, backupInfo) { if (error && error.reason === BackupsError.NOT_FOUND) return callback(new AppsError(AppsError.EXTERNAL_ERROR, error.message)); if (error && error.reason === BackupsError.EXTERNAL_ERROR) return callback(new AppsError(AppsError.EXTERNAL_ERROR, error.message)); if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error)); - if (!restoreConfig) callback(new AppsError(AppsError.EXTERNAL_ERROR, 'Could not get restore config')); + if (!backupInfo.manifest) callback(new AppsError(AppsError.EXTERNAL_ERROR, 'Could not get restore config')); // re-validate because this new box version may not accept old configs - error = checkManifestConstraints(restoreConfig.manifest); + error = checkManifestConstraints(backupInfo.manifest); if (error) return callback(error); var values = { lastBackupId: data.backupId || null, // when null, apptask simply reinstalls - manifest: restoreConfig.manifest, + manifest: backupInfo.manifest, oldConfig: getAppConfig(app) }; @@ -815,24 +815,24 @@ function clone(appId, data, auditSource, callback) { if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND)); if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error)); - backups.getRestoreConfig(backupId, function (error, restoreConfig) { + backups.get(backupId, function (error, backupInfo) { if (error && error.reason === BackupsError.EXTERNAL_ERROR) return callback(new AppsError(AppsError.EXTERNAL_ERROR, error.message)); if (error && error.reason === BackupsError.NOT_FOUND) return callback(new AppsError(AppsError.EXTERNAL_ERROR, error.message)); if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error)); - if (!restoreConfig) callback(new AppsError(AppsError.EXTERNAL_ERROR, 'Could not get restore config')); + if (!backupInfo.manifest) callback(new AppsError(AppsError.EXTERNAL_ERROR, 'Could not get restore config')); // re-validate because this new box version may not accept old configs - error = checkManifestConstraints(restoreConfig.manifest); + error = checkManifestConstraints(backupInfo.manifest); if (error) return callback(error); error = validateHostname(location, config.fqdn()); if (error) return callback(error); - error = validatePortBindings(portBindings, restoreConfig.manifest.tcpPorts); + error = validatePortBindings(portBindings, backupInfo.manifest.tcpPorts); if (error) return callback(error); - var newAppId = uuid.v4(), appStoreId = app.appStoreId, manifest = restoreConfig.manifest; + var newAppId = uuid.v4(), appStoreId = app.appStoreId, manifest = backupInfo.manifest; appstore.purchase(newAppId, appStoreId, function (error) { if (error && error.reason === AppstoreError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND)); diff --git a/src/backupdb.js b/src/backupdb.js index 6bd257520..08fbb6b1e 100644 --- a/src/backupdb.js +++ b/src/backupdb.js @@ -6,7 +6,7 @@ var assert = require('assert'), safe = require('safetydance'), util = require('util'); -var BACKUPS_FIELDS = [ 'id', 'creationTime', 'version', 'type', 'dependsOn', 'state', 'restoreConfigJson', 'format' ]; +var BACKUPS_FIELDS = [ 'id', 'creationTime', 'version', 'type', 'dependsOn', 'state', 'manifestJson', 'format' ]; exports = module.exports = { add: add, @@ -34,8 +34,8 @@ function postProcess(result) { result.dependsOn = result.dependsOn ? result.dependsOn.split(',') : [ ]; - result.restoreConfig = result.restoreConfigJson ? safe.JSON.parse(result.restoreConfigJson) : null; - delete result.restoreConfigJson; + result.manifest = result.manifestJson ? safe.JSON.parse(result.manifestJson) : null; + delete result.manifestJson; } function getByTypeAndStatePaged(type, state, page, perPage, callback) { @@ -109,15 +109,15 @@ function add(backup, callback) { assert.strictEqual(typeof backup.version, 'string'); assert(backup.type === exports.BACKUP_TYPE_APP || backup.type === exports.BACKUP_TYPE_BOX); assert(util.isArray(backup.dependsOn)); - assert.strictEqual(typeof backup.restoreConfig, 'object'); + assert.strictEqual(typeof backup.manifest, 'object'); assert.strictEqual(typeof backup.format, 'string'); assert.strictEqual(typeof callback, 'function'); var creationTime = backup.creationTime || new Date(); // allow tests to set the time - var restoreConfig = backup.restoreConfig ? JSON.stringify(backup.restoreConfig) : ''; + var manifestJson = JSON.stringify(backup.manifest); - database.query('INSERT INTO backups (id, version, type, creationTime, state, dependsOn, restoreConfigJson, format) VALUES (?, ?, ?, ?, ?, ?, ?, ?)', - [ backup.id, backup.version, backup.type, creationTime, exports.BACKUP_STATE_NORMAL, backup.dependsOn.join(','), restoreConfig, backup.format ], + database.query('INSERT INTO backups (id, version, type, creationTime, state, dependsOn, manifestJson, format) VALUES (?, ?, ?, ?, ?, ?, ?, ?)', + [ backup.id, backup.version, backup.type, creationTime, exports.BACKUP_STATE_NORMAL, backup.dependsOn.join(','), manifestJson, backup.format ], function (error) { if (error && error.code === 'ER_DUP_ENTRY') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS)); if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error)); diff --git a/src/backups.js b/src/backups.js index a6ee0d2ed..d0b81ea14 100644 --- a/src/backups.js +++ b/src/backups.js @@ -8,7 +8,7 @@ exports = module.exports = { getByStatePaged: getByStatePaged, getByAppIdPaged: getByAppIdPaged, - getRestoreConfig: getRestoreConfig, + get: get, ensureBackup: ensureBackup, @@ -150,16 +150,15 @@ function getByAppIdPaged(page, perPage, appId, callback) { }); } -function getRestoreConfig(backupId, callback) { +function get(backupId, callback) { assert.strictEqual(typeof backupId, 'string'); assert.strictEqual(typeof callback, 'function'); backupdb.get(backupId, function (error, result) { if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new BackupsError(BackupsError.NOT_FOUND, error)); if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error)); - if (!result.restoreConfig) return callback(new BackupsError(BackupsError.NOT_FOUND, error)); - callback(null, result.restoreConfig); + callback(null, result); }); } @@ -559,7 +558,7 @@ function rotateBoxBackup(backupConfig, timestamp, appBackupIds, callback) { log(`Rotating box backup to id ${backupId}`); - backupdb.add({ id: backupId, version: config.version(), type: backupdb.BACKUP_TYPE_BOX, dependsOn: appBackupIds, restoreConfig: null, format: format }, function (error) { + backupdb.add({ id: backupId, version: config.version(), type: backupdb.BACKUP_TYPE_BOX, dependsOn: appBackupIds, manifest: null, format: format }, function (error) { if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error)); var copy = api(backupConfig.provider).copy(backupConfig, getBackupFilePath(backupConfig, 'snapshot/box', format), getBackupFilePath(backupConfig, backupId, format)); @@ -616,17 +615,14 @@ function snapshotApp(app, manifest, callback) { log(`Snapshotting app ${app.id}`); - var restoreConfig = apps.getAppConfig(app); - restoreConfig.manifest = manifest; - - if (!safe.fs.writeFileSync(path.join(paths.APPS_DATA_DIR, app.id + '/config.json'), JSON.stringify(restoreConfig))) { + if (!safe.fs.writeFileSync(path.join(paths.APPS_DATA_DIR, app.id + '/config.json'), JSON.stringify(apps.getAppConfig(app)))) { return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, 'Error creating config.json: ' + safe.error.message)); } addons.backupAddons(app, manifest.addons, function (error) { if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message)); - return callback(null, restoreConfig); + return callback(null); }); } @@ -653,14 +649,13 @@ function rotateAppBackup(backupConfig, app, timestamp, callback) { if (!snapshotInfo) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, 'Snapshot info missing or corrupt')); var snapshotTime = snapshotInfo.timestamp.replace(/[T.]/g, '-').replace(/[:Z]/g,''); - var restoreConfig = snapshotInfo.restoreConfig; - var manifest = restoreConfig.manifest; + var manifest = snapshotInfo.restoreConfig ? snapshotInfo.restoreConfig.manifest : snapshotInfo.manifest; // compat var backupId = util.format('%s/app_%s_%s_v%s', timestamp, app.id, snapshotTime, manifest.version); const format = backupConfig.format; log(`Rotating app backup of ${app.id} to id ${backupId}`); - backupdb.add({ id: backupId, version: manifest.version, type: backupdb.BACKUP_TYPE_APP, dependsOn: [ ], restoreConfig: restoreConfig, format: format }, function (error) { + backupdb.add({ id: backupId, version: manifest.version, type: backupdb.BACKUP_TYPE_APP, dependsOn: [ ], manifest: manifest, format: format }, function (error) { if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error)); var copy = api(backupConfig.provider).copy(backupConfig, getBackupFilePath(backupConfig, `snapshot/app_${app.id}`, format), getBackupFilePath(backupConfig, backupId, format)); @@ -694,7 +689,7 @@ function uploadAppSnapshot(backupConfig, app, manifest, callback) { var startTime = new Date(); - snapshotApp(app, manifest, function (error, restoreConfig) { + snapshotApp(app, manifest, function (error) { if (error) return callback(error); var backupId = util.format('snapshot/app_%s', app.id); @@ -704,7 +699,7 @@ function uploadAppSnapshot(backupConfig, app, manifest, callback) { debugApp(app, 'uploadAppSnapshot: %s done time: %s secs', backupId, (new Date() - startTime)/1000); - setSnapshotInfo(app.id, { timestamp: new Date().toISOString(), restoreConfig: restoreConfig, format: backupConfig.format }, callback); + setSnapshotInfo(app.id, { timestamp: new Date().toISOString(), manifest: manifest, format: backupConfig.format }, callback); }); }); } diff --git a/src/test/backups-test.js b/src/test/backups-test.js index 531fdf24a..28640faec 100644 --- a/src/test/backups-test.js +++ b/src/test/backups-test.js @@ -124,7 +124,7 @@ describe('backups', function () { version: '1.0.0', type: backupdb.BACKUP_TYPE_BOX, dependsOn: [ 'backup-app-00', 'backup-app-01' ], - restoreConfig: null, + manifest: null, format: 'tgz' }; @@ -133,7 +133,7 @@ describe('backups', function () { version: '1.0.0', type: backupdb.BACKUP_TYPE_APP, dependsOn: [], - restoreConfig: null, + manifest: null, format: 'tgz' }; @@ -142,7 +142,7 @@ describe('backups', function () { version: '1.0.0', type: backupdb.BACKUP_TYPE_APP, dependsOn: [], - restoreConfig: null, + manifest: null, format: 'tgz' }; @@ -151,7 +151,7 @@ describe('backups', function () { version: '1.0.0', type: backupdb.BACKUP_TYPE_BOX, dependsOn: [ 'backup-app-10', 'backup-app-11' ], - restoreConfig: null, + manifest: null, format: 'tgz' }; @@ -160,7 +160,7 @@ describe('backups', function () { version: '1.0.0', type: backupdb.BACKUP_TYPE_APP, dependsOn: [], - restoreConfig: null, + manifest: null, format: 'tgz' }; @@ -169,7 +169,7 @@ describe('backups', function () { version: '1.0.0', type: backupdb.BACKUP_TYPE_APP, dependsOn: [], - restoreConfig: null, + manifest: null, format: 'tgz' }; diff --git a/src/test/database-test.js b/src/test/database-test.js index 6067d2b96..05c068f0f 100644 --- a/src/test/database-test.js +++ b/src/test/database-test.js @@ -1027,7 +1027,7 @@ describe('database', function () { version: '1.0.0', type: backupdb.BACKUP_TYPE_BOX, dependsOn: [ 'dep1' ], - restoreConfig: null, + manifest: null, format: 'tgz' }; @@ -1044,7 +1044,7 @@ describe('database', function () { expect(result.type).to.be(backupdb.BACKUP_TYPE_BOX); expect(result.creationTime).to.be.a(Date); expect(result.dependsOn).to.eql(['dep1']); - expect(result.restoreConfig).to.eql(null); + expect(result.manifest).to.eql(null); done(); }); }); @@ -1067,7 +1067,7 @@ describe('database', function () { expect(results[0].id).to.be('backup-box'); expect(results[0].version).to.be('1.0.0'); expect(results[0].dependsOn).to.eql(['dep1']); - expect(results[0].restoreConfig).to.eql(null); + expect(results[0].manifest).to.eql(null); done(); }); @@ -1093,7 +1093,7 @@ describe('database', function () { version: '1.0.0', type: backupdb.BACKUP_TYPE_APP, dependsOn: [ ], - restoreConfig: { manifest: { foo: 'bar' } }, + manifest: { foo: 'bar' }, format: 'tgz' }; @@ -1110,7 +1110,7 @@ describe('database', function () { expect(result.type).to.be(backupdb.BACKUP_TYPE_APP); expect(result.creationTime).to.be.a(Date); expect(result.dependsOn).to.eql([]); - expect(result.restoreConfig).to.eql({ manifest: { foo: 'bar' } }); + expect(result.manifest).to.eql({ foo: 'bar' }); done(); }); }); @@ -1124,7 +1124,7 @@ describe('database', function () { expect(results[0].id).to.be('app_appid_123'); expect(results[0].version).to.be('1.0.0'); expect(results[0].dependsOn).to.eql([]); - expect(results[0].restoreConfig).to.eql({ manifest: { foo: 'bar' } }); + expect(results[0].manifest).to.eql({ foo: 'bar' }); done(); });