mail: keep mail backups separately from box backups

part of #717
2021-09-26 18:37:04 -07:00
parent 184fc70e97
commit ad3dbe8daa
10 changed files with 165 additions and 68 deletions
@@ -1,13 +1,16 @@
 'use strict';

 exports = module.exports = {
-    backupBoxAndApps,
+    fullBackup,

    restore,

    backupApp,
    downloadApp,

+    backupMail,
+    downloadMail,
+
    upload,

    _restoreFsMetadata: restoreFsMetadata,
@@ -750,11 +753,39 @@ async function uploadBoxSnapshot(backupConfig, progressCallback) {
    await backups.setSnapshotInfo('box', { timestamp: new Date().toISOString(), format: backupConfig.format });
 }

-async function rotateBoxBackup(backupConfig, tag, options, appBackupIds, progressCallback) {
+async function copy(backupConfig, sourceBackupId, destBackupId, options, progressCallback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof sourceBackupId, 'string');
+    assert.strictEqual(typeof destBackupId, 'string');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    const format = backupConfig.format;
+
+    return new Promise((resolve, reject) => {
+        const startTime = new Date();
+
+        const copyEvents = storage.api(backupConfig.provider).copy(backupConfig, storage.getBackupFilePath(backupConfig, sourceBackupId, format), storage.getBackupFilePath(backupConfig, destBackupId, format));
+        copyEvents.on('progress', (message) => progressCallback({ message }));
+        copyEvents.on('done', async function (copyBackupError) {
+            const state = copyBackupError ? backups.BACKUP_STATE_ERROR : backups.BACKUP_STATE_NORMAL;
+
+            const [error] = await safe(backups.update(destBackupId, { preserveSecs: options.preserveSecs || 0, state }));
+            if (copyBackupError) return reject(copyBackupError);
+            if (error) return reject(error);
+
+            debug(`copy: copied successfully to id ${destBackupId}. Took ${(new Date() - startTime)/1000} seconds`);
+
+            resolve();
+        });
+    });
+}
+
+async function rotateBoxBackup(backupConfig, tag, options, dependsOn, progressCallback) {
    assert.strictEqual(typeof backupConfig, 'object');
    assert.strictEqual(typeof tag, 'string');
    assert.strictEqual(typeof options, 'object');
-    assert(Array.isArray(appBackupIds));
+    assert(Array.isArray(dependsOn));
    assert.strictEqual(typeof progressCallback, 'function');

    const backupId = `${tag}/box_v${constants.VERSION}`;
@@ -767,33 +798,19 @@ async function rotateBoxBackup(backupConfig, tag, options, appBackupIds, progres
        packageVersion: constants.VERSION,
        type: backups.BACKUP_TYPE_BOX,
        state: backups.BACKUP_STATE_CREATING,
-        identifier: 'box',
-        dependsOn: appBackupIds,
+        identifier: backups.BACKUP_IDENTIFIER_BOX,
+        dependsOn,
        manifest: null,
-        format: format
+        format
    };

    await backups.add(backupId, data);
-
-    return new Promise((resolve, reject) => {
-        const copy = storage.api(backupConfig.provider).copy(backupConfig, storage.getBackupFilePath(backupConfig, 'snapshot/box', format), storage.getBackupFilePath(backupConfig, backupId, format));
-        copy.on('progress', (message) => progressCallback({ message: `box: ${message}` }));
-        copy.on('done', async function (copyBackupError) {
-            const state = copyBackupError ? backups.BACKUP_STATE_ERROR : backups.BACKUP_STATE_NORMAL;
-
-            const [error] = await safe(backups.update(backupId, { preserveSecs: options.preserveSecs || 0, state }));
-            if (copyBackupError) return reject(copyBackupError);
-            if (error) return reject(error);
-
-            debug(`rotateBoxBackup: rotated successfully as id ${backupId}`);
-
-            resolve(backupId);
-        });
-    });
+    await copy(backupConfig, 'snapshot/box', backupId, options, progressCallback);
+    return backupId;
 }

-async function backupBoxWithAppBackupIds(appBackupIds, tag, options, progressCallback) {
-    assert(Array.isArray(appBackupIds));
+async function backupBox(dependsOn, tag, options, progressCallback) {
+    assert(Array.isArray(dependsOn));
    assert.strictEqual(typeof tag, 'string');
    assert.strictEqual(typeof options, 'object');
    assert.strictEqual(typeof progressCallback, 'function');
@@ -802,7 +819,7 @@ async function backupBoxWithAppBackupIds(appBackupIds, tag, options, progressCal

    await uploadBoxSnapshot(backupConfig, progressCallback);

-    const backupId = await rotateBoxBackup(backupConfig, tag, options, appBackupIds, progressCallback);
+    const backupId = await rotateBoxBackup(backupConfig, tag, options, dependsOn, progressCallback);
    return backupId;
 }

@@ -813,8 +830,6 @@ async function rotateAppBackup(backupConfig, app, tag, options, progressCallback
    assert.strictEqual(typeof options, 'object');
    assert.strictEqual(typeof progressCallback, 'function');

-    const startTime = new Date();
-
    const snapshotInfo = backups.getSnapshotInfo(app.id);

    const manifest = snapshotInfo.restoreConfig ? snapshotInfo.restoreConfig.manifest : snapshotInfo.manifest; // compat
@@ -835,22 +850,8 @@ async function rotateAppBackup(backupConfig, app, tag, options, progressCallback
    };

    await backups.add(backupId, data);
-
-    return new Promise((resolve, reject) => {
-        const copy = storage.api(backupConfig.provider).copy(backupConfig, storage.getBackupFilePath(backupConfig, `snapshot/app_${app.id}`, format), storage.getBackupFilePath(backupConfig, backupId, format));
-        copy.on('progress', (message) => progressCallback({ message: `${message} (${app.fqdn})` }));
-        copy.on('done', async function (copyBackupError) {
-            const state = copyBackupError ? backups.BACKUP_STATE_ERROR : backups.BACKUP_STATE_NORMAL;
-
-            const [error] = await safe(backups.update(backupId, { preserveSecs: options.preserveSecs || 0, state }));
-            if (copyBackupError) return reject(copyBackupError);
-            if (error) return reject(error);
-
-            debug(`rotateAppBackup: rotated ${app.fqdn} to id ${backupId}. Took ${(new Date() - startTime)/1000} seconds`);
-
-            resolve(backupId);
-        });
-    });
+    await copy(backupConfig,  `snapshot/app_${app.id}`, backupId, options, progressCallback);
+    return backupId;
 }

 async function backupApp(app, options, progressCallback) {
@@ -931,8 +932,98 @@ async function backupAppWithTag(app, tag, options, progressCallback) {
    return backupId;
 }

-// this function expects you to have a lock. Unlike other progressCallback this also has a progress field
-async function backupBoxAndApps(options, progressCallback) {
+async function uploadMailSnapshot(backupConfig, progressCallback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    const mailDataDir = safe.fs.realpathSync(paths.MAIL_DATA_DIR);
+    if (!mailDataDir) throw new BoxError(BoxError.FS_ERROR, `Error resolving maildata: ${safe.error.message}`);
+
+    const uploadConfig = {
+        backupId: 'snapshot/mail',
+        backupConfig,
+        dataLayout: new DataLayout(mailDataDir, []),
+        progressTag: 'mail'
+    };
+
+    progressCallback({ message: 'Uploading mail snapshot' });
+
+    const startTime = new Date();
+
+    await util.promisify(runBackupUpload)(uploadConfig, progressCallback);
+
+    debug(`uploadMailSnapshot: took ${(new Date() - startTime)/1000} seconds`);
+
+    await backups.setSnapshotInfo('mail', { timestamp: new Date().toISOString(), format: backupConfig.format });
+}
+
+async function rotateMailBackup(backupConfig, tag, options, progressCallback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof tag, 'string');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    const backupId = `${tag}/mail_v${constants.VERSION}`;
+    const format = backupConfig.format;
+
+    debug(`rotateMailBackup: rotating to id ${backupId}`);
+
+    const data = {
+        encryptionVersion: backupConfig.encryption ? 2 : null,
+        packageVersion: constants.VERSION,
+        type: backups.BACKUP_TYPE_MAIL,
+        state: backups.BACKUP_STATE_CREATING,
+        identifier: backups.BACKUP_IDENTIFIER_MAIL,
+        dependsOn: [],
+        manifest: null,
+        format: format
+    };
+
+    await backups.add(backupId, data);
+    await copy(backupConfig, 'snapshot/mail', backupId, options, progressCallback);
+    return backupId;
+}
+
+async function backupMailWithTag(tag, options, progressCallback) {
+    assert.strictEqual(typeof tag, 'string');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    debug(`backupMailWithTag: backing up mail with tag ${tag}`);
+
+    const backupConfig = await settings.getBackupConfig();
+    await uploadMailSnapshot(backupConfig, progressCallback);
+    const backupId = await rotateMailBackup(backupConfig, tag, options, progressCallback);
+    return backupId;
+}
+
+async function backupMail(options, progressCallback) {
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    const tag = (new Date()).toISOString().replace(/[T.]/g, '-').replace(/[:Z]/g,'');
+    debug(`backupMail: backing up mail with tag ${tag}`);
+    return await backupMailWithTag(tag, options, progressCallback);
+}
+
+async function downloadMail(restoreConfig, progressCallback) {
+    assert.strictEqual(typeof restoreConfig, 'object');
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    const mailDataDir = safe.fs.realpathSync(paths.MAIL_DATA_DIR);
+    if (!mailDataDir) throw new BoxError(BoxError.FS_ERROR, `Error resolving maildata: ${safe.error.message}`);
+    const dataLayout = new DataLayout(mailDataDir, []);
+
+    const startTime = new Date();
+    const backupConfig = restoreConfig.backupConfig || await settings.getBackupConfig();
+
+    const downloadAsync = util.promisify(download);
+    await downloadAsync(backupConfig, restoreConfig.backupId, restoreConfig.backupFormat, dataLayout, progressCallback);
+    debug('downloadMail: time: %s', (new Date() - startTime)/1000);
+}
+
+// this function is called from external process. calling process is expected to have a lock
+async function fullBackup(options, progressCallback) {
    assert.strictEqual(typeof options, 'object');
    assert.strictEqual(typeof progressCallback, 'function');

@@ -941,7 +1032,7 @@ async function backupBoxAndApps(options, progressCallback) {
    const allApps = await apps.list();

    let percent = 1;
-    let step = 100/(allApps.length+2);
+    let step = 100/(allApps.length+3);

    const appBackupIds = [];
    for (const app of allApps) {
@@ -949,19 +1040,24 @@ async function backupBoxAndApps(options, progressCallback) {
        percent += step;

        if (!app.enableBackup) {
-            debug(`Skipped backup ${app.fqdn}`);
+            debug(`fullBackup: skipped backup ${app.fqdn}`);
            return; // nothing to backup
        }

        const startTime = new Date();
-        const backupId = await backupAppWithTag(app, tag, options, (progress) => progressCallback({ percent: percent, message: progress.message }));
-        debug(`backupBoxAndApps: app ${app.fqdn} backup finished. Took ${(new Date() - startTime)/1000} seconds`);
-        if (backupId) appBackupIds.push(backupId); // backupId can be null if in BAD_STATE and never backed up
+        const appBackupId = await backupAppWithTag(app, tag, options, (progress) => progressCallback({ percent: percent, message: progress.message }));
+        debug(`fullBackup: app ${app.fqdn} backup finished. Took ${(new Date() - startTime)/1000} seconds`);
+        if (appBackupId) appBackupIds.push(appBackupId); // backupId can be null if in BAD_STATE and never backed up
    }

+    progressCallback({ percent: percent, message: 'Backing up mail' });
+    percent += step;
+    const mailBackupId = await backupMailWithTag(tag, options, (progress) => progressCallback({ percent: percent, message: progress.message }));
+
    progressCallback({ percent: percent, message: 'Backing up system data' });
    percent += step;

-    const backupId = await backupBoxWithAppBackupIds(appBackupIds, tag, options, (progress) => progressCallback({ percent: percent, message: progress.message }));
+    const dependsOn = appBackupIds.concat(mailBackupId);
+    const backupId = await backupBox(dependsOn, tag, options, (progress) => progressCallback({ percent: percent, message: progress.message }));
    return backupId;
 }