jjkiers/cloudron-box
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

archives: use separate table

Browse Source 
Cleaner to separate things from the backups table.

* icon, appConfig, appStoreIcon etc are only valid for archives
* older version cloudron does not have appConfig in backups table (so it
  cannot be an archive entry)
This commit is contained in:
Girish Ramakrishnan
2024-12-10 10:06:52 +01:00
parent 2ad93c114e
commit 490840b71d
12 changed files with 261 additions and 122 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
migrations/20241209150823-archives-add-table.js Normal file
View File

@@ -0,0 +1,20 @@
'use strict';
exports.up = async function (db) {
const cmd = 'CREATE TABLE archives(' +
'id VARCHAR(128) NOT NULL UNIQUE,' +
'backupId VARCHAR(128) NOT NULL,' +
'creationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,' +
'appStoreIcon MEDIUMBLOB,' +
'icon MEDIUMBLOB,' +
'appConfigJson TEXT,' +
'FOREIGN KEY(backupId) REFERENCES backups(id),' +
'PRIMARY KEY (id)) ' +
'CHARACTER SET utf8 COLLATE utf8_bin';
await db.runSql(cmd);
};
exports.down = async function (db) {
await db.runSql('DROP TABLE archives');
};

16
migrations/20241209150823-backups-add-archive.js
View File

@@ -1,16 +0,0 @@
'use strict';
exports.up = async function(db) {
await db.runSql('ALTER TABLE backups ADD COLUMN archive BOOLEAN DEFAULT 0');
await db.runSql('ALTER TABLE backups ADD COLUMN icon MEDIUMBLOB');
await db.runSql('ALTER TABLE backups ADD COLUMN appStoreIcon MEDIUMBLOB');
await db.runSql('ALTER TABLE backups ADD COLUMN appConfigJson TEXT');
};
exports.down = async function(db) {
await db.runSql('ALTER TABLE backups DROP COLUMN archive');
await db.runSql('ALTER TABLE backups DROP COLUMN icon');
await db.runSql('ALTER TABLE backups DROP COLUMN appStoreIcon');
await db.runSql('ALTER TABLE backups DROP COLUMN appConfigJson');
};

15
migrations/schema.sql
View File

@@ -156,14 +156,21 @@ CREATE TABLE IF NOT EXISTS backups(
manifestJson TEXT, /* to validate if the app can be installed in this version of box */
format VARCHAR(16) DEFAULT "tgz",
preserveSecs INTEGER DEFAULT 0,
archive BOOLEAN DEFAULT 0,
appStoreIcon MEDIUMBLOB, /* only valid with archive */
icon MEDIUMBLOB, /* only valid with archive */
appConfigJson TEXT, /* only valid with archive */
INDEX creationTime_index (creationTime),
PRIMARY KEY (id));
CREATE TABLE IF NOT EXISTS archives(
id VARCHAR(128) NOT NULL UNIQUE,
backupId VARCHAR(128) NOT NULL,
creationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
appStoreIcon MEDIUMBLOB,
icon MEDIUMBLOB,
appConfigJson TEXT,
FOREIGN KEY(backupId) REFERENCES backups(id),
PRIMARY KEY (id));
CREATE TABLE IF NOT EXISTS eventlog(
id VARCHAR(128) NOT NULL,
action VARCHAR(128) NOT NULL,

4
src/apps.js
View File

@@ -146,6 +146,7 @@ exports = module.exports = {
const appstore = require('./appstore.js'),
appTaskManager = require('./apptaskmanager.js'),
archives = require('./archives.js'),
assert = require('assert'),
backups = require('./backups.js'),
BoxError = require('./boxerror.js'),
@@ -2496,8 +2497,7 @@ async function archive(app, backupId, auditSource) {
const icons = await getIcons(app.id);
const { taskId } = await uninstall(app, auditSource);
await backups.update(result[0].id, { archive: true, icon: icons.icon, appStoreIcon: icons.appStoreIcon });
if (!result[0].appConfig) await backups.update(result[0].id, { appConfig: app }); // workaround for previous versions not setting appConfig
await archives.add(backupId, { icon: icons.icon, appStoreIcon: icons.appStoreIcon, appConfig: app });
return { taskId };
}

98
src/archives.js Normal file
View File

@@ -0,0 +1,98 @@
'use strict';
exports = module.exports = {
get,
getIcon,
add,
list,
listBackupIds,
del
};
const assert = require('assert'),
BoxError = require('./boxerror.js'),
database = require('./database.js'),
safe = require('safetydance'),
uuid = require('uuid');
const ARCHIVE_FIELDS = [ 'id', 'backupId', 'creationTime', 'appConfigJson', '(icon IS NOT NULL) AS hasIcon', '(appStoreIcon IS NOT NULL) AS hasAppStoreIcon' ];
function postProcess(result) {
assert.strictEqual(typeof result, 'object');
result.appConfig = result.appConfigJson ? safe.JSON.parse(result.appConfigJson) : null;
delete result.appConfigJson;
return result;
}
async function get(id) {
assert.strictEqual(typeof id, 'string');
const result = await database.query(`SELECT ${ARCHIVE_FIELDS} FROM archives WHERE id = ? ORDER BY creationTime DESC`, [ id ]);
if (result.length === 0) return null;
return postProcess(result[0]);
}
async function getIcons(id) {
assert.strictEqual(typeof id, 'string');
const results = await database.query('SELECT icon, appStoreIcon FROM archives WHERE id=?', [ id ]);
if (results.length === 0) return null;
return { icon: results[0].icon, appStoreIcon: results[0].appStoreIcon };
}
async function getIcon(id, options) {
assert.strictEqual(typeof id, 'string');
assert.strictEqual(typeof options, 'object');
const icons = await getIcons(id);
if (!icons) throw new BoxError(BoxError.NOT_FOUND, 'No such backup');
if (!options.original && icons.icon) return icons.icon;
if (icons.appStoreIcon) return icons.appStoreIcon;
return null;
}
async function add(backupId, data) {
assert.strictEqual(typeof backupId, 'string');
assert.strictEqual(typeof data, 'object');
assert(data.appConfig && typeof data.appConfig === 'object');
const id = uuid.v4();
const [error] = await safe(database.query('INSERT INTO archives (id, backupId, icon, appStoreIcon, appConfigJson) VALUES (?, ?, ?, ?, ?)',
[ id, backupId, data.icon, data.appStoreIcon, JSON.stringify(data.appConfig) ]));
if (error && error.code === 'ER_NO_REFERENCED_ROW_2') throw new BoxError(BoxError.NOT_FOUND, 'Backup not found');
if (error && error.code === 'ER_DUP_ENTRY') throw new BoxError(BoxError.ALREADY_EXISTS, 'Archive already exists');
if (error) throw error;
return id;
}
async function list(page, perPage) {
assert(typeof page === 'number' && page > 0);
assert(typeof perPage === 'number' && perPage > 0);
const results = await database.query(`SELECT ${ARCHIVE_FIELDS} FROM archives ORDER BY creationTime DESC LIMIT ?,?`, [ (page-1)*perPage, perPage ]);
results.forEach(function (result) { postProcess(result); });
return results;
}
async function listBackupIds() {
const results = await database.query(`SELECT backupId FROM archives`, []);
return results.map(r => r.backupId);
}
async function del(id, auditSource) {
assert.strictEqual(typeof id, 'string');
assert(auditSource && typeof auditSource === 'object');
const result = await database.query('DELETE FROM archives WHERE id=?', [ id ]);
if (result.affectedRows !== 1) throw new BoxError(BoxError.NOT_FOUND, 'No such archive');
}

12
src/backupcleaner.js
View File

@@ -9,6 +9,7 @@ exports = module.exports = {
};
const apps = require('./apps.js'),
archives = require('./archives.js'),
assert = require('assert'),
backupFormat = require('./backupformat.js'),
backups = require('./backups.js'),
@@ -28,14 +29,12 @@ function applyBackupRetention(allBackups, retention, referencedBackupIds) {
const now = new Date();
for (const backup of allBackups) {
if (backup.archive) {
backup.keepReason = 'archive';
} else if (backup.state === backups.BACKUP_STATE_ERROR) {
if (backup.state === backups.BACKUP_STATE_ERROR) {
backup.discardReason = 'error';
} else if (backup.state === backups.BACKUP_STATE_CREATING) {
if ((now - backup.creationTime) < 48*60*60*1000) backup.keepReason = 'creating';
else backup.discardReason = 'creating-too-long';
} else if (referencedBackupIds.includes(backup.id)) {
} else if (referencedBackupIds.includes(backup.id)) { // could also be in archives
backup.keepReason = 'referenced';
} else if ((backup.preserveSecs === -1) || ((now - backup.creationTime) < (backup.preserveSecs * 1000))) {
backup.keepReason = 'preserveSecs';
@@ -76,7 +75,7 @@ function applyBackupRetention(allBackups, retention, referencedBackupIds) {
}
for (const backup of allBackups) {
debug(`applyBackupRetentionPolicy: ${backup.remotePath} keep/discard: ${backup.keepReason || backup.discardReason || 'unprocessed'}`);
debug(`applyBackupRetention: ${backup.remotePath} keep/discard: ${backup.keepReason || backup.discardReason || 'unprocessed'}`);
}
}
@@ -297,7 +296,8 @@ async function run(progressCallback) {
const removedMailBackupPaths = await cleanupMailBackups(backupConfig, retention, referencedBackupIds, progressCallback);
await progressCallback({ percent: 40, message: 'Cleaning app backups' });
const removedAppBackupPaths = await cleanupAppBackups(backupConfig, retention, referencedBackupIds, progressCallback);
const archivedBackupIds = await archives.listBackupIds();
const removedAppBackupPaths = await cleanupAppBackups(backupConfig, retention, referencedBackupIds.concat(archivedBackupIds), progressCallback);
await progressCallback({ percent: 70, message: 'Checking storage backend and removing stale entries in database' });
const missingBackupPaths = await cleanupMissingBackups(backupConfig, progressCallback);

71
src/backups.js
View File

@@ -10,13 +10,6 @@ exports = module.exports = {
list,
del,
archives: {
get: archivesGet,
getIcon: archivesGetIcon,
list: archivesList,
del: archivesDel
},
startBackupTask,
startCleanupTask,
@@ -80,7 +73,7 @@ const assert = require('assert'),
tasks = require('./tasks.js'),
_ = require('underscore');
const BACKUPS_FIELDS = [ 'id', 'remotePath', 'label', 'identifier', 'creationTime', 'packageVersion', 'type', 'dependsOnJson', 'state', 'manifestJson', 'format', 'preserveSecs', 'encryptionVersion', 'archive', 'appConfigJson' ];
const BACKUPS_FIELDS = [ 'id', 'remotePath', 'label', 'identifier', 'creationTime', 'packageVersion', 'type', 'dependsOnJson', 'state', 'manifestJson', 'format', 'preserveSecs', 'encryptionVersion' ];
function postProcess(result) {
assert.strictEqual(typeof result, 'object');
@@ -91,9 +84,6 @@ function postProcess(result) {
result.manifest = result.manifestJson ? safe.JSON.parse(result.manifestJson) : null;
delete result.manifestJson;
result.appConfig = result.appConfigJson ? safe.JSON.parse(result.appConfigJson) : null;
delete result.appConfigJson;
return result;
}
@@ -137,10 +127,9 @@ async function add(data) {
const manifestJson = JSON.stringify(data.manifest);
const prefixId = data.type === exports.BACKUP_TYPE_APP ? `${data.type}_${data.identifier}` : data.type; // type and identifier are same for other types
const id = `${prefixId}_v${data.packageVersion}_${hat(32)}`; // id is used by the UI to derive dependent packages. making this a UUID will require a lot of db querying
const appConfigJson = 'appConfig' in data ? JSON.stringify(data.appConfig) : null;
const [error] = await safe(database.query('INSERT INTO backups (id, remotePath, identifier, encryptionVersion, packageVersion, type, creationTime, state, dependsOnJson, manifestJson, format, preserveSecs, appConfigJson) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
[ id, data.remotePath, data.identifier, data.encryptionVersion, data.packageVersion, data.type, creationTime, data.state, JSON.stringify(data.dependsOn), manifestJson, data.format, data.preserveSecs, appConfigJson ]));
const [error] = await safe(database.query('INSERT INTO backups (id, remotePath, identifier, encryptionVersion, packageVersion, type, creationTime, state, dependsOnJson, manifestJson, format, preserveSecs) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
[ id, data.remotePath, data.identifier, data.encryptionVersion, data.packageVersion, data.type, creationTime, data.state, JSON.stringify(data.dependsOn), manifestJson, data.format, data.preserveSecs ]));
if (error && error.code === 'ER_DUP_ENTRY') throw new BoxError(BoxError.ALREADY_EXISTS, 'Backup already exists');
if (error) throw error;
@@ -182,55 +171,6 @@ async function getByTypePaged(type, page, perPage) {
return results;
}
async function archivesGet(id) {
assert.strictEqual(typeof id, 'string');
const result = await database.query(`SELECT ${BACKUPS_FIELDS} FROM backups WHERE id = ? AND archive = 1 ORDER BY creationTime DESC`, [ id ]);
if (result.length === 0) return null;
return postProcess(result[0]);
}
async function archivesGetIcons(id) {
assert.strictEqual(typeof id, 'string');
const results = await database.query('SELECT icon, appStoreIcon FROM backups WHERE id = ?', [ id ]);
if (results.length === 0) return null;
return { icon: results[0].icon, appStoreIcon: results[0].appStoreIcon };
}
async function archivesGetIcon(id, options) {
assert.strictEqual(typeof id, 'string');
assert.strictEqual(typeof options, 'object');
const icons = await archivesGetIcons(id);
if (!icons) throw new BoxError(BoxError.NOT_FOUND, 'No such backup');
if (!options.original && icons.icon) return icons.icon;
if (icons.appStoreIcon) return icons.appStoreIcon;
return null;
}
async function archivesList(page, perPage) {
assert(typeof page === 'number' && page > 0);
assert(typeof perPage === 'number' && perPage > 0);
const results = await database.query(`SELECT ${BACKUPS_FIELDS} FROM backups WHERE archive = 1 ORDER BY creationTime DESC LIMIT ?,?`, [ (page-1)*perPage, perPage ]);
results.forEach(function (result) { postProcess(result); });
return results;
}
async function archivesDel(id, auditSource) {
assert.strictEqual(typeof id, 'string');
assert(auditSource && typeof auditSource === 'object');
const result = await database.query('UPDATE backups SET archive = 0 WHERE id=?', [ id ]);
if (result.affectedRows !== 1) throw new BoxError(BoxError.NOT_FOUND, 'Backup not found in archive');
}
function validateLabel(label) {
assert.strictEqual(typeof label, 'string');
@@ -269,12 +209,9 @@ async function update(id, data) {
const fields = [], values = [];
for (const p in data) {
if (p === 'label' || p === 'preserveSecs' || p === 'archive' || p === 'icon' || p === 'appStoreIcon') {
if (p === 'label' || p === 'preserveSecs') {
fields.push(p + ' = ?');
values.push(data[p]);
} else if (p === 'appConfig') {
fields.push(`${p}Json = ?`);
values.push(JSON.stringify(data[p]));
}
}
values.push(id);

2
src/routes/archives.js
View File

@@ -10,7 +10,7 @@ exports = module.exports = {
};
const assert = require('assert'),
{ archives } = require('../backups.js'),
archives = require('../archives.js'),
AuditSource = require('../auditsource.js'),
BoxError = require('../boxerror.js'),
HttpError = require('connect-lastmile').HttpError,

24
src/routes/test/archives-test.js
View File

@@ -5,7 +5,8 @@
'use strict';
const backups = require('../../backups.js'),
const archives = require('../../archives.js'),
backups = require('../../backups.js'),
common = require('./common.js'),
expect = require('expect.js'),
superagent = require('superagent');
@@ -13,7 +14,7 @@ const backups = require('../../backups.js'),
describe('Archives API', function () {
const { setup, cleanup, serverUrl, owner } = common;
const nonArchiveBackup = {
const appBackup = {
id: null,
remotePath: 'app_appid_123',
encryptionVersion: null,
@@ -26,16 +27,15 @@ describe('Archives API', function () {
format: 'tgz',
preserveSecs: 0,
label: '',
archive: false
};
const archiveBackup = Object.assign({}, nonArchiveBackup, {archive: true, remotePath: 'app_appid_234'});
const appConfig = { loc: 'loc1' };
let archiveId;
before(async function () {
await setup();
nonArchiveBackup.id = await backups.add(nonArchiveBackup);
archiveBackup.id = await backups.add(archiveBackup);
await backups.update(archiveBackup.id, { archive: true });
appBackup.id = await backups.add(appBackup);
archiveId = await archives.add(appBackup.id, { appConfig });
});
after(cleanup);
@@ -44,14 +44,14 @@ describe('Archives API', function () {
.query({ access_token: owner.token });
expect(response.statusCode).to.equal(200);
expect(response.body.archives.length).to.be(1);
expect(response.body.archives[0].id).to.be(archiveBackup.id);
expect(response.body.archives[0].id).to.be(archiveId);
});
it('get valid archive', async function () {
const response = await superagent.get(`${serverUrl}/api/v1/archives/${archiveBackup.id}`)
const response = await superagent.get(`${serverUrl}/api/v1/archives/${archiveId}`)
.query({ access_token: owner.token });
expect(response.statusCode).to.equal(200);
expect(response.body.remotePath).to.be('app_appid_234');
expect(response.body.appConfig).to.eql(appConfig);
});
it('cannot get invalid archive', async function () {
@@ -62,14 +62,14 @@ describe('Archives API', function () {
});
it('cannot del invalid archive', async function () {
const response = await superagent.del(`${serverUrl}/api/v1/archives/${nonArchiveBackup.id}`)
const response = await superagent.del(`${serverUrl}/api/v1/archives/random`)
.query({ access_token: owner.token })
.ok(() => true);
expect(response.statusCode).to.equal(404);
});
it('del valid archive', async function () {
const response = await superagent.del(`${serverUrl}/api/v1/archives/${archiveBackup.id}`)
const response = await superagent.del(`${serverUrl}/api/v1/archives/${archiveId}`)
.query({ access_token: owner.token });
expect(response.statusCode).to.equal(204);

84
src/test/archives-test.js Normal file
View File

@@ -0,0 +1,84 @@
/* global it:false */
/* global describe:false */
/* global before:false */
/* global after:false */
'use strict';
const archives = require('../archives.js'),
backups = require('../backups.js'),
BoxError = require('../boxerror.js'),
common = require('./common.js'),
expect = require('expect.js'),
safe = require('safetydance');
describe('Archives', function () {
const { setup, cleanup, auditSource } = common;
const appBackup = {
id: null,
remotePath: 'backup-box',
encryptionVersion: 2,
packageVersion: '1.0.0',
type: backups.BACKUP_TYPE_APP,
state: backups.BACKUP_STATE_NORMAL,
identifier: 'box',
dependsOn: [ 'dep1' ],
manifest: null,
format: 'tgz',
preserveSecs: 0,
label: '',
};
before(async function () {
await setup();
appBackup.id = await backups.add(appBackup);
});
after(cleanup);
const appConfig = { loc: 'loc1' };
let archiveId;
it('cannot add bad backup to archives', async function () {
const [error] = await safe(archives.add('badId', { appConfig }));
expect(error.reason).to.be(BoxError.NOT_FOUND);
});
it('can add good backup to archives', async function () {
archiveId = await archives.add(appBackup.id, { appConfig });
});
it('cannot get invalid archive', async function () {
const result = await archives.get('bad');
expect(result).to.be(null);
});
it('can get archive', async function () {
const result = await archives.get(archiveId);
expect(result.appConfig).to.eql(appConfig);
});
it('can list archives', async function () {
const result = await archives.list(1, 100);
expect(result.length).to.be(1);
expect(result[0].id).to.be(archiveId);
expect(result[0].appConfig).to.eql(appConfig);
});
it('can list backupIds', async function () {
const result = await archives.listBackupIds();
expect(result.length).to.be(1);
expect(result[0]).to.be(appBackup.id);
});
it('cannot delete bad archive', async function () {
const [error] = await safe(archives.del('badId', auditSource));
expect(error.reason).to.be(BoxError.NOT_FOUND);
});
it('can del valid archive', async function () {
await archives.del(archiveId, auditSource);
const result = await archives.list(1, 10);
expect(result.length).to.be(0);
});
});

33
src/test/backupcleaner-test.js
View File

@@ -6,7 +6,8 @@
'use strict';
const backupCleaner = require('../backupcleaner.js'),
const archives = require('../archives.js'),
backupCleaner = require('../backupcleaner.js'),
backups = require('../backups.js'),
common = require('./common.js'),
expect = require('expect.js'),
@@ -33,7 +34,6 @@ describe('backup cleaner', function () {
manifest: null,
format: 'tgz',
preserveSecs: 0,
archive: false
};
describe('retention', function () {
@@ -123,12 +123,6 @@ describe('backup cleaner', function () {
expect(b[8].keepReason).to.be('keepMonthly');
expect(b[9].keepReason).to.be(undefined);
});
it('keeps archive', function () {
const backup = Object.assign({}, backupTemplate, { archive: true });
backupCleaner._applyBackupRetention([backup], { keepWithinSecs: 0, keepLatest: false }, []);
expect(backup.keepReason).to.be('archive');
});
});
describe('task', function () {
@@ -216,6 +210,22 @@ describe('backup cleaner', function () {
preserveSecs: 0
};
const BACKUP_2_APP_2 = { // this is archived and left alone
id: null,
remotePath: 'backup-app-2',
encryptionVersion: null,
packageVersion: '2.0.0',
type: backups.BACKUP_TYPE_APP,
state: backups.BACKUP_STATE_NORMAL,
identifier: 'app2',
dependsOn: [],
manifest: null,
format: 'tgz',
preserveSecs: 0
};
const app2Config = { loc: 'apploc2' };
before(async function () {
await settings._set(settings.BACKUP_STORAGE_KEY, JSON.stringify({
provider: 'filesystem',
@@ -231,7 +241,6 @@ describe('backup cleaner', function () {
console.log('started task', taskId);
// eslint-disable-next-line no-constant-condition
while (true) {
await timers.setTimeout(1000);
@@ -260,6 +269,9 @@ describe('backup cleaner', function () {
BACKUP_1_APP_1.id = await backups.add(BACKUP_1_APP_1);
BACKUP_1_BOX.dependsOn = [ BACKUP_1_APP_0.id, BACKUP_1_APP_1.id ];
BACKUP_1_BOX.id = await backups.add(BACKUP_1_BOX);
BACKUP_2_APP_2.id = await backups.add(BACKUP_2_APP_2);
await archives.add(BACKUP_2_APP_2.id, { appConfig: app2Config });
});
it('succeeds with box backups, keeps latest', async function () {
@@ -296,11 +308,12 @@ describe('backup cleaner', function () {
await cleanupBackups();
let result = await backups.getByTypePaged(backups.BACKUP_TYPE_APP, 1, 1000);
expect(result.length).to.equal(3);
expect(result.length).to.equal(4);
result = result.sort((r1, r2) => r1.remotePath.localeCompare(r2.remotePath));
expect(result[0].id).to.be(BACKUP_0_APP_0.id); // because app is installed, latest backup is preserved
expect(result[1].id).to.be(BACKUP_1_APP_0.id); // referenced by box
expect(result[2].id).to.be(BACKUP_1_APP_1.id); // referenced by box
expect(result[3].id).to.be(BACKUP_2_APP_2.id); // referenced by archive
});
});
});

4
src/test/backups-test.js
View File

@@ -31,8 +31,6 @@ describe('backups', function () {
format: 'tgz',
preserveSecs: 0,
label: '',
archive: false,
appConfig: null
};
const appBackup = {
@@ -48,8 +46,6 @@ describe('backups', function () {
format: 'tgz',
preserveSecs: 0,
label: '',
archive: false,
appConfig: null
};
describe('crud', function () {