jjkiers/cloudron-box
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix the backup tests

Browse Source
This commit is contained in:
Girish Ramakrishnan
2021-07-14 19:03:12 -07:00
parent 004e812d60
commit 2840bba4bf
8 changed files with 585 additions and 623 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/backuptask.js
+32 -42
View File
@@ -376,10 +376,9 @@ function sync(backupConfig, backupId, dataLayout, progressCallback, callback) {
}
// this is not part of 'snapshotting' because we need root access to traverse
function saveFsMetadata(dataLayout, metadataFile, callback) {
async function saveFsMetadata(dataLayout, metadataFile) {
assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
assert.strictEqual(typeof metadataFile, 'string');
assert.strictEqual(typeof callback, 'function');
// contains paths prefixed with './'
let metadata = {
@@ -391,24 +390,22 @@ function saveFsMetadata(dataLayout, metadataFile, callback) {
// we assume small number of files. spawnSync will raise a ENOBUFS error after maxBuffer
for (let lp of dataLayout.localPaths()) {
const emptyDirs = safe.child_process.execSync(`find ${lp} -type d -empty`, { encoding: 'utf8', maxBuffer: 1024 * 1024 * 30 });
if (emptyDirs === null) return callback(new BoxError(BoxError.FS_ERROR, `Error finding empty dirs: ${safe.error.message}`));
if (emptyDirs === null) throw new BoxError(BoxError.FS_ERROR, `Error finding empty dirs: ${safe.error.message}`);
if (emptyDirs.length) metadata.emptyDirs = metadata.emptyDirs.concat(emptyDirs.trim().split('\n').map((ed) => dataLayout.toRemotePath(ed)));
const execFiles = safe.child_process.execSync(`find ${lp} -type f -executable`, { encoding: 'utf8', maxBuffer: 1024 * 1024 * 30 });
if (execFiles === null) return callback(new BoxError(BoxError.FS_ERROR, `Error finding executables: ${safe.error.message}`));
if (execFiles === null) throw new BoxError(BoxError.FS_ERROR, `Error finding executables: ${safe.error.message}`);
if (execFiles.length) metadata.execFiles = metadata.execFiles.concat(execFiles.trim().split('\n').map((ef) => dataLayout.toRemotePath(ef)));
const symlinks = safe.child_process.execSync(`find ${lp} -type l`, { encoding: 'utf8', maxBuffer: 1024 * 1024 * 30 });
if (symlinks === null) return callback(new BoxError(BoxError.FS_ERROR, `Error finding symlinks: ${safe.error.message}`));
if (symlinks === null) throw new BoxError(BoxError.FS_ERROR, `Error finding symlinks: ${safe.error.message}`);
if (symlinks.length) metadata.symlinks = metadata.symlinks.concat(symlinks.trim().split('\n').map((sl) => {
const target = safe.fs.readlinkSync(sl);
return { path: dataLayout.toRemotePath(sl), target };
}));
}
if (!safe.fs.writeFileSync(metadataFile, JSON.stringify(metadata, null, 4))) return callback(new BoxError(BoxError.FS_ERROR, `Error writing fs metadata: ${safe.error.message}`));
callback();
if (!safe.fs.writeFileSync(metadataFile, JSON.stringify(metadata, null, 4))) throw new BoxError(BoxError.FS_ERROR, `Error writing fs metadata: ${safe.error.message}`);
}
// this function is called via backupupload (since it needs root to traverse app's directory)
@@ -448,7 +445,7 @@ function upload(backupId, format, dataLayoutString, progressCallback, callback)
}, callback);
} else {
async.series([
saveFsMetadata.bind(null, dataLayout, `${dataLayout.localRoot()}/fsmetadata.json`),
saveFsMetadata(dataLayout, `${dataLayout.localRoot()}/fsmetadata.json`),
sync.bind(null, backupConfig, backupId, dataLayout, progressCallback)
], callback);
}
@@ -512,43 +509,35 @@ function tarExtract(inStream, dataLayout, encryption, callback) {
callback(null, ps);
}
function restoreFsMetadata(dataLayout, metadataFile, callback) {
async function restoreFsMetadata(dataLayout, metadataFile) {
assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
assert.strictEqual(typeof metadataFile, 'string');
assert.strictEqual(typeof callback, 'function');
debug(`Recreating empty directories in ${dataLayout.toString()}`);
var metadataJson = safe.fs.readFileSync(metadataFile, 'utf8');
if (metadataJson === null) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Error loading fsmetadata.json:' + safe.error.message));
if (metadataJson === null) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Error loading fsmetadata.json:' + safe.error.message);
var metadata = safe.JSON.parse(metadataJson);
if (metadata === null) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Error parsing fsmetadata.json:' + safe.error.message));
if (metadata === null) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Error parsing fsmetadata.json:' + safe.error.message);
async.eachSeries(metadata.emptyDirs, function createPath(emptyDir, iteratorDone) {
fs.mkdir(dataLayout.toLocalPath(emptyDir), { recursive: true }, iteratorDone);
}, function (error) {
if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `unable to create path: ${error.message}`));
for (const emptyDir of metadata.emptyDirs) {
const [mkdirError] = await safe(fs.promises.mkdir(dataLayout.toLocalPath(emptyDir), { recursive: true }));
if (mkdirError) throw new BoxError(BoxError.FS_ERROR, `unable to create path: ${mkdirError.message}`);
}
async.eachSeries(metadata.execFiles, function createPath(execFile, iteratorDone) {
fs.chmod(dataLayout.toLocalPath(execFile), parseInt('0755', 8), iteratorDone);
}, function (error) {
if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `unable to chmod: ${error.message}`));
for (const execFile of metadata.execFiles) {
const [chmodError] = await safe(fs.promises.chmod(dataLayout.toLocalPath(execFile), parseInt('0755', 8)));
if (chmodError) throw new BoxError(BoxError.FS_ERROR, `unable to chmod: ${chmodError.message}`);
}
async.eachSeries(metadata.symlinks || [], function createSymlink(symlink, iteratorDone) {
if (!symlink.target) return iteratorDone();
// the path may not exist if we had a directory full of symlinks
fs.mkdir(path.dirname(dataLayout.toLocalPath(symlink.path)), { recursive: true }, function (error) {
if (error) return iteratorDone(error);
fs.symlink(symlink.target, dataLayout.toLocalPath(symlink.path), 'file', iteratorDone);
});
}, function (error) {
if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `unable to symlink: ${error.message}`));
callback();
});
});
});
for (const symlink of (metadata.symlinks || [])) {
if (!symlink.target) continue;
// the path may not exist if we had a directory full of symlinks
const [mkdirError] = await safe(fs.promises.mkdir(path.dirname(dataLayout.toLocalPath(symlink.path)), { recursive: true }));
if (mkdirError) throw new BoxError(BoxError.FS_ERROR, `unable to symlink (mkdir): ${mkdirError.message}`);
const [symlinkError] = await safe(fs.promises.symlink(symlink.target, dataLayout.toLocalPath(symlink.path), 'file'));
if (symlinkError) throw new BoxError(BoxError.FS_ERROR, `unable to symlink: ${symlinkError.message}`);
}
}
function downloadDir(backupConfig, backupFilePath, dataLayout, progressCallback, callback) {
@@ -646,10 +635,11 @@ function download(backupConfig, backupId, format, dataLayout, progressCallback,
});
}, callback);
} else {
downloadDir(backupConfig, backupFilePath, dataLayout, progressCallback, function (error) {
downloadDir(backupConfig, backupFilePath, dataLayout, progressCallback, async function (error) {
if (error) return callback(error);
restoreFsMetadata(dataLayout, `${dataLayout.localRoot()}/fsmetadata.json`, callback);
[error] = await safe(restoreFsMetadata(dataLayout, `${dataLayout.localRoot()}/fsmetadata.json`));
callback(error);
});
}
}
@@ -843,8 +833,8 @@ function backupBoxWithAppBackupIds(appBackupIds, tag, options, progressCallback,
uploadBoxSnapshot(backupConfig, progressCallback, async function (error) {
if (error) return callback(error);
const [rotateError] = await safe(rotateBoxBackup(backupConfig, tag, options, appBackupIds, progressCallback));
callback(rotateError);
const [rotateError, backupId] = await safe(rotateBoxBackup(backupConfig, tag, options, appBackupIds, progressCallback));
callback(rotateError, backupId);
});
});
}
@@ -990,8 +980,8 @@ async function backupAppWithTag(app, tag, options, progressCallback, callback) {
uploadAppSnapshot(backupConfig, app, progressCallback, async function (error) {
if (error) return callback(error);
const [rotateError] = await safe(rotateAppBackup(backupConfig, app, tag, options, progressCallback));
callback(rotateError);
const [rotateError, backupId] = await safe(rotateAppBackup(backupConfig, app, tag, options, progressCallback));
callback(rotateError, backupId);
});
});
}