Girish Ramakrishnan
23c3263562
in the previous approach, we used to clear the taskId after the integrity check completes. for one, we lose track of the task (to show the logs). for another, we have to clear these taskId on platform startup to handle crashes. in the new approach, we keep the taskId and use the task's active flag to determine if task is active.
272 lines
11 KiB
JavaScript
272 lines
11 KiB
JavaScript
import assert from 'node:assert';
|
|
import BoxError from './boxerror.js';
|
|
import database from './database.js';
|
|
import debugModule from 'debug';
|
|
import eventlog from './eventlog.js';
|
|
import hat from './hat.js';
|
|
import safe from 'safetydance';
|
|
import tasks from './tasks.js';
|
|
|
|
const debug = debugModule('box:backups');
|
|
|
|
const BACKUP_TYPE_APP = 'app';
|
|
const BACKUP_STATE_NORMAL = 'normal';
|
|
|
|
|
|
const BACKUPS_FIELDS = [ 'id', 'remotePath', 'label', 'identifier', 'creationTime', 'packageVersion', 'type', 'integrityJson',
|
|
'statsJson', 'dependsOnJson', 'state', 'manifestJson', 'preserveSecs', 'encryptionVersion', 'appConfigJson', 'siteId',
|
|
'integrityCheckTaskId', 'lastIntegrityCheckTime', 'integrityCheckStatus', 'integrityCheckResultJson' ].join(',');
|
|
|
|
function postProcess(result) {
|
|
assert.strictEqual(typeof result, 'object');
|
|
|
|
result.dependsOn = result.dependsOnJson ? safe.JSON.parse(result.dependsOnJson) : [];
|
|
delete result.dependsOnJson;
|
|
|
|
result.manifest = result.manifestJson ? safe.JSON.parse(result.manifestJson) : null;
|
|
delete result.manifestJson;
|
|
|
|
result.integrity = result.integrityJson ? safe.JSON.parse(result.integrityJson) : null;
|
|
delete result.integrityJson;
|
|
|
|
result.stats = result.statsJson ? safe.JSON.parse(result.statsJson) : null;
|
|
delete result.statsJson;
|
|
|
|
result.appConfig = result.appConfigJson ? safe.JSON.parse(result.appConfigJson) : null;
|
|
delete result.appConfigJson;
|
|
|
|
result.integrityCheckResult = result.integrityCheckResultJson ? safe.JSON.parse(result.integrityCheckResultJson) : null;
|
|
delete result.integrityCheckResultJson;
|
|
|
|
return result;
|
|
}
|
|
|
|
function removePrivateFields(backup) {
|
|
delete backup.integrityCheckTaskId;
|
|
return backup;
|
|
}
|
|
|
|
async function attachIntegrityTaskInfo(backup) {
|
|
backup.integrityCheckTask = backup.integrityCheckTaskId ? await tasks.get(String(backup.integrityCheckTaskId)) : null;
|
|
}
|
|
|
|
async function add(data) {
|
|
assert(data && typeof data === 'object');
|
|
assert.strictEqual(typeof data.remotePath, 'string');
|
|
assert(data.encryptionVersion === null || typeof data.encryptionVersion === 'number');
|
|
assert.strictEqual(typeof data.packageVersion, 'string');
|
|
assert.strictEqual(typeof data.type, 'string');
|
|
assert.strictEqual(typeof data.identifier, 'string');
|
|
assert.strictEqual(typeof data.state, 'string');
|
|
assert(Array.isArray(data.dependsOn));
|
|
assert.strictEqual(typeof data.manifest, 'object');
|
|
assert.strictEqual(typeof data.preserveSecs, 'number');
|
|
assert.strictEqual(typeof data.appConfig, 'object');
|
|
assert.strictEqual(typeof data.siteId, 'string');
|
|
|
|
const creationTime = data.creationTime || new Date(); // allow tests to set the time
|
|
const manifestJson = JSON.stringify(data.manifest);
|
|
const prefixId = data.type === BACKUP_TYPE_APP ? `${data.type}_${data.identifier}` : data.type; // type and identifier are same for other types
|
|
const id = `${prefixId}_v${data.packageVersion}_${hat(32)}`; // id is used by the UI to derive dependent packages. making this a UUID will require a lot of db querying
|
|
const appConfigJson = data.appConfig ? JSON.stringify(data.appConfig) : null;
|
|
const statsJson = data.stats ? JSON.stringify(data.stats) : null;
|
|
const integrityJson = data.integrity ? JSON.stringify(data.integrity) : null;
|
|
|
|
const [error] = await safe(database.query('INSERT INTO backups (id, remotePath, identifier, encryptionVersion, packageVersion, type, creationTime, state, dependsOnJson, manifestJson, preserveSecs, appConfigJson, siteId, statsJson, integrityJson) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
|
|
[ id, data.remotePath, data.identifier, data.encryptionVersion, data.packageVersion, data.type, creationTime, data.state, JSON.stringify(data.dependsOn), manifestJson, data.preserveSecs, appConfigJson, data.siteId, statsJson, integrityJson ]));
|
|
|
|
if (error && error.sqlCode === 'ER_DUP_ENTRY') throw new BoxError(BoxError.ALREADY_EXISTS, 'Backup already exists');
|
|
if (error) throw error;
|
|
|
|
return id;
|
|
}
|
|
|
|
async function getLatestInTargetByIdentifier(identifier, siteId) {
|
|
assert.strictEqual(typeof identifier, 'string');
|
|
assert.strictEqual(typeof siteId, 'string');
|
|
|
|
const results = await database.query(`SELECT ${BACKUPS_FIELDS} FROM backups WHERE identifier = ? AND state = ? AND siteId = ? LIMIT 1`, [ identifier, BACKUP_STATE_NORMAL, siteId ]);
|
|
if (!results.length) return null;
|
|
|
|
await attachIntegrityTaskInfo(results[0]);
|
|
return postProcess(results[0]);
|
|
}
|
|
|
|
async function get(id) {
|
|
assert.strictEqual(typeof id, 'string');
|
|
|
|
const results = await database.query(`SELECT ${BACKUPS_FIELDS} FROM backups WHERE id = ? ORDER BY creationTime DESC`, [ id ]);
|
|
if (results.length === 0) return null;
|
|
|
|
await attachIntegrityTaskInfo(results[0]);
|
|
return postProcess(results[0]);
|
|
}
|
|
|
|
function validateLabel(label) {
|
|
assert.strictEqual(typeof label, 'string');
|
|
|
|
if (label.length >= 200) return new BoxError(BoxError.BAD_FIELD, 'label too long');
|
|
if (/[^a-zA-Z0-9._() -]/.test(label)) return new BoxError(BoxError.BAD_FIELD, 'label can only contain alphanumerals, space, dot, hyphen, brackets or underscore');
|
|
|
|
return null;
|
|
}
|
|
|
|
// this is called by REST API
|
|
async function update(backup, data) {
|
|
assert.strictEqual(typeof backup, 'object');
|
|
assert.strictEqual(typeof data, 'object');
|
|
|
|
let error;
|
|
if ('label' in data) {
|
|
error = validateLabel(data.label);
|
|
if (error) throw error;
|
|
}
|
|
|
|
const fields = [], values = [];
|
|
for (const p in data) {
|
|
if (p === 'label' || p === 'preserveSecs' || p === 'state') {
|
|
fields.push(p + ' = ?');
|
|
values.push(data[p]);
|
|
} else if (p === 'stats') {
|
|
fields.push(`${p}Json=?`);
|
|
values.push(JSON.stringify(data[p]));
|
|
}
|
|
}
|
|
values.push(backup.id);
|
|
|
|
const result = await database.query('UPDATE backups SET ' + fields.join(', ') + ' WHERE id = ?', values);
|
|
if (result.affectedRows !== 1) throw new BoxError(BoxError.NOT_FOUND, 'Backup not found');
|
|
|
|
if ('preserveSecs' in data) {
|
|
// update the dependancies
|
|
for (const depId of backup.dependsOn) {
|
|
await database.query('UPDATE backups SET preserveSecs=? WHERE id = ?', [ data.preserveSecs, depId]);
|
|
}
|
|
}
|
|
}
|
|
|
|
async function listByTypePaged(type, siteId, page, perPage) {
|
|
assert.strictEqual(typeof type, 'string');
|
|
assert.strictEqual(typeof siteId, 'string');
|
|
assert(typeof page === 'number' && page > 0);
|
|
assert(typeof perPage === 'number' && perPage > 0);
|
|
|
|
const results = await database.query(`SELECT ${BACKUPS_FIELDS} FROM backups WHERE siteId=? AND type = ? ORDER BY creationTime DESC LIMIT ?,?`, [ siteId, type, (page-1)*perPage, perPage ]);
|
|
|
|
for (const r of results) {
|
|
await attachIntegrityTaskInfo(r);
|
|
postProcess(r);
|
|
}
|
|
|
|
return results;
|
|
}
|
|
|
|
async function listByIdentifierAndStatePaged(identifier, state, page, perPage) {
|
|
assert.strictEqual(typeof identifier, 'string');
|
|
assert.strictEqual(typeof state, 'string');
|
|
assert(typeof page === 'number' && page > 0);
|
|
assert(typeof perPage === 'number' && perPage > 0);
|
|
|
|
const results = await database.query(`SELECT ${BACKUPS_FIELDS} FROM backups WHERE identifier = ? AND state = ? ORDER BY creationTime DESC LIMIT ?,?`, [ identifier, state, (page-1)*perPage, perPage ]);
|
|
|
|
for (const r of results) {
|
|
await attachIntegrityTaskInfo(r);
|
|
postProcess(r);
|
|
}
|
|
|
|
return results;
|
|
}
|
|
|
|
async function del(id) {
|
|
assert.strictEqual(typeof id, 'string');
|
|
|
|
const result = await database.query('DELETE FROM backups WHERE id=?', [ id ]);
|
|
if (result.affectedRows !== 1) throw new BoxError(BoxError.NOT_FOUND, 'Backup not found');
|
|
}
|
|
|
|
async function setIntegrityResult(backup, status, result) {
|
|
assert.strictEqual(typeof backup, 'object');
|
|
assert.strictEqual(typeof status, 'string');
|
|
assert.strictEqual(typeof result, 'object');
|
|
|
|
const now = new Date();
|
|
|
|
await database.query('UPDATE backups SET lastIntegrityCheckTime = ?, integrityCheckStatus = ?, integrityCheckResultJson = ? WHERE id = ?',
|
|
[ now, status, JSON.stringify(result), backup.id ]);
|
|
}
|
|
|
|
async function startIntegrityCheck(backup, auditSource) {
|
|
assert.strictEqual(typeof backup, 'object');
|
|
assert.strictEqual(typeof auditSource, 'object');
|
|
|
|
const ids = [backup.id, ...backup.dependsOn];
|
|
const placeholders = ids.map(() => '?').join(',');
|
|
|
|
const taskId = await tasks.add(tasks.TASK_CHECK_BACKUP_INTEGRITY, [ backup.id ]);
|
|
|
|
const didUpdate = await database.runInTransaction(async (query) => {
|
|
const rows = await query(`SELECT id, integrityCheckTaskId FROM backups WHERE id IN (${placeholders}) FOR UPDATE`, [...ids]);
|
|
for (const b of rows) {
|
|
if (!b.integrityCheckTaskId || b.integrityCheckTaskId === backup.integrityCheckTaskId) continue;
|
|
const t = await tasks.get(String(b.integrityCheckTaskId));
|
|
if (t?.active) return false;
|
|
}
|
|
await query(`UPDATE backups SET integrityCheckTaskId = ?, lastIntegrityCheckTime = ?, integrityCheckStatus = ?, integrityCheckResultJson = ? WHERE id IN (${placeholders})`, [taskId, null, null, null, ...ids]);
|
|
return true;
|
|
});
|
|
|
|
if (!didUpdate) throw new BoxError(BoxError.CONFLICT, 'An integrity check is already in progress for a dependent backup');
|
|
|
|
await eventlog.add(eventlog.ACTION_BACKUP_INTEGRITY_START, auditSource, { taskId, backupId: backup.id });
|
|
|
|
// background
|
|
tasks.startTask(taskId, {})
|
|
.then(async (status) => {
|
|
debug(`startIntegrityCheck: task completed`);
|
|
await eventlog.add(eventlog.ACTION_BACKUP_INTEGRITY_FINISH, auditSource, { status, taskId, backupId: backup.id });
|
|
})
|
|
.catch(async (error) => {
|
|
debug(`startIntegrityCheck: task error. ${error.message}`);
|
|
await eventlog.add(eventlog.ACTION_BACKUP_INTEGRITY_FINISH, auditSource, { errorMessage: error.message, taskId, backupId: backup.id });
|
|
});
|
|
|
|
return taskId;
|
|
}
|
|
|
|
async function stopIntegrityCheck(backup, auditSource) {
|
|
assert.strictEqual(typeof backup, 'object');
|
|
assert.strictEqual(typeof auditSource, 'object');
|
|
|
|
const task = backup.integrityCheckTaskId ? await tasks.get(String(backup.integrityCheckTaskId)) : null;
|
|
if (!task?.active) throw new BoxError(BoxError.BAD_STATE, 'task is not active');
|
|
|
|
await tasks.stopTask(backup.integrityCheckTaskId);
|
|
}
|
|
|
|
export default {
|
|
get,
|
|
listByIdentifierAndStatePaged,
|
|
getLatestInTargetByIdentifier, // brutal function name
|
|
add,
|
|
update,
|
|
listByTypePaged,
|
|
del,
|
|
|
|
removePrivateFields,
|
|
|
|
startIntegrityCheck,
|
|
stopIntegrityCheck,
|
|
setIntegrityResult,
|
|
|
|
BACKUP_IDENTIFIER_BOX: 'box',
|
|
BACKUP_IDENTIFIER_MAIL: 'mail',
|
|
|
|
BACKUP_TYPE_APP,
|
|
BACKUP_TYPE_BOX: 'box',
|
|
BACKUP_TYPE_MAIL: 'mail',
|
|
|
|
BACKUP_STATE_NORMAL,
|
|
BACKUP_STATE_CREATING: 'creating',
|
|
BACKUP_STATE_ERROR: 'error',
|
|
};
|