reorder functions for no-use-before-define
This commit is contained in:
Girish Ramakrishnan
+85
-89
@@ -21,10 +21,6 @@ import util from 'node:util';
|
||||
const debug = debugModule('box:backupformat/rsync');
|
||||
const shell = shellModule('backupformat/rsync');
|
||||
|
||||
const _saveFsMetadata = saveFsMetadata;
|
||||
const _restoreFsMetadata = restoreFsMetadata;
|
||||
|
||||
|
||||
async function addFile(sourceFile, encryption, uploader, progressCallback) {
|
||||
assert.strictEqual(typeof sourceFile, 'string');
|
||||
assert.strictEqual(typeof encryption, 'object');
|
||||
@@ -71,6 +67,89 @@ async function addFile(sourceFile, encryption, uploader, progressCallback) {
|
||||
};
|
||||
}
|
||||
|
||||
async function saveFsMetadata(dataLayout, metadataFile) {
|
||||
assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
|
||||
assert.strictEqual(typeof metadataFile, 'string');
|
||||
|
||||
// contains paths prefixed with './'
|
||||
const metadata = {
|
||||
emptyDirs: [],
|
||||
execFiles: [],
|
||||
symlinks: []
|
||||
};
|
||||
|
||||
// we assume small number of files. spawnSync will raise a ENOBUFS error after maxBuffer
|
||||
for (const lp of dataLayout.localPaths()) {
|
||||
const [emptyDirsError, emptyDirs] = await safe(shell.spawn('find', [lp, '-type', 'd', '-empty'], { encoding: 'utf8', maxLines: 80000 }));
|
||||
if (emptyDirsError && emptyDirsError.stdoutLineCount >= 80000) throw new BoxError(BoxError.FS_ERROR, `Too many empty directories. Run "find ${lp} -type d -empty" to investigate`);
|
||||
if (emptyDirsError) throw emptyDirsError;
|
||||
if (emptyDirs.length) metadata.emptyDirs = metadata.emptyDirs.concat(emptyDirs.trim().split('\n').map((ed) => dataLayout.toRemotePath(ed)));
|
||||
|
||||
const [execFilesError, execFiles] = await safe(shell.spawn('find', [lp, '-type', 'f', '-executable'], { encoding: 'utf8', maxLines: 20000 }));
|
||||
if (execFilesError && execFilesError.stdoutLineCount >= 20000) throw new BoxError(BoxError.FS_ERROR, `Too many executable files. Run "find ${lp} -type f -executable" to investigate`);
|
||||
if (execFilesError) throw execFilesError;
|
||||
if (execFiles.length) metadata.execFiles = metadata.execFiles.concat(execFiles.trim().split('\n').map((ef) => dataLayout.toRemotePath(ef)));
|
||||
|
||||
const [symlinkFilesError, symlinkFiles] = await safe(shell.spawn('find', [lp, '-type', 'l'], { encoding: 'utf8', maxLines: 20000 }));
|
||||
if (symlinkFilesError && symlinkFilesError.stdoutLineCount >= 20000) throw new BoxError(BoxError.FS_ERROR, `Too many symlinks. Run "find ${lp} -type l" to investigate`);
|
||||
if (symlinkFilesError) throw symlinkFilesError;
|
||||
|
||||
if (symlinkFiles.length) metadata.symlinks = metadata.symlinks.concat(symlinkFiles.trim().split('\n').map((sl) => {
|
||||
const site = safe.fs.readlinkSync(sl);
|
||||
return { path: dataLayout.toRemotePath(sl), site };
|
||||
}));
|
||||
}
|
||||
|
||||
if (!safe.fs.writeFileSync(metadataFile, JSON.stringify(metadata, null, 2))) throw new BoxError(BoxError.FS_ERROR, `Error writing fs metadata: ${safe.error.message}`);
|
||||
}
|
||||
|
||||
// this is not part of 'snapshotting' because we need root access to traverse
|
||||
async function restoreFsMetadata(dataLayout, metadataFile) {
|
||||
assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
|
||||
assert.strictEqual(typeof metadataFile, 'string');
|
||||
|
||||
debug(`Recreating empty directories in ${dataLayout.toString()}`);
|
||||
|
||||
const metadataJson = safe.fs.readFileSync(metadataFile, 'utf8');
|
||||
if (metadataJson === null) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Error loading fsmetadata.json:' + safe.error.message);
|
||||
const metadata = safe.JSON.parse(metadataJson);
|
||||
if (metadata === null) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Error parsing fsmetadata.json:' + safe.error.message);
|
||||
|
||||
for (const emptyDir of metadata.emptyDirs) {
|
||||
const [mkdirError] = await safe(fs.promises.mkdir(dataLayout.toLocalPath(emptyDir), { recursive: true }));
|
||||
if (mkdirError) throw new BoxError(BoxError.FS_ERROR, `unable to create path: ${mkdirError.message}`);
|
||||
}
|
||||
|
||||
for (const execFile of metadata.execFiles) {
|
||||
const [chmodError] = await safe(fs.promises.chmod(dataLayout.toLocalPath(execFile), parseInt('0755', 8)));
|
||||
if (chmodError) throw new BoxError(BoxError.FS_ERROR, `unable to chmod: ${chmodError.message}`);
|
||||
}
|
||||
|
||||
for (const symlink of (metadata.symlinks || [])) {
|
||||
if (!symlink.site) continue;
|
||||
// the path may not exist if we had a directory full of symlinks
|
||||
const [mkdirError] = await safe(fs.promises.mkdir(path.dirname(dataLayout.toLocalPath(symlink.path)), { recursive: true }));
|
||||
if (mkdirError) throw new BoxError(BoxError.FS_ERROR, `unable to symlink (mkdir): ${mkdirError.message}`);
|
||||
const [symlinkError] = await safe(fs.promises.symlink(symlink.site, dataLayout.toLocalPath(symlink.path), 'file'));
|
||||
if (symlinkError) throw new BoxError(BoxError.FS_ERROR, `unable to symlink: ${symlinkError.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function copy(backupSite, fromPath, toPath, progressCallback) {
|
||||
assert.strictEqual(typeof backupSite, 'object');
|
||||
assert.strictEqual(typeof fromPath, 'string');
|
||||
assert.strictEqual(typeof toPath, 'string');
|
||||
assert.strictEqual(typeof progressCallback, 'function');
|
||||
|
||||
await backupSites.storageApi(backupSite).copyDir(backupSite.config, backupSite.limits, fromPath, toPath, progressCallback);
|
||||
}
|
||||
|
||||
function getFileExtension(encryption) {
|
||||
assert.strictEqual(typeof encryption, 'boolean');
|
||||
|
||||
return ''; // this also signals to backupcleanear that we are dealing with directories
|
||||
}
|
||||
|
||||
async function sync(backupSite, remotePath, dataLayout, progressCallback) {
|
||||
assert.strictEqual(typeof backupSite, 'object');
|
||||
assert.strictEqual(typeof remotePath, 'string');
|
||||
@@ -136,74 +215,6 @@ async function sync(backupSite, remotePath, dataLayout, progressCallback) {
|
||||
};
|
||||
}
|
||||
|
||||
// this is not part of 'snapshotting' because we need root access to traverse
|
||||
async function saveFsMetadata(dataLayout, metadataFile) {
|
||||
assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
|
||||
assert.strictEqual(typeof metadataFile, 'string');
|
||||
|
||||
// contains paths prefixed with './'
|
||||
const metadata = {
|
||||
emptyDirs: [],
|
||||
execFiles: [],
|
||||
symlinks: []
|
||||
};
|
||||
|
||||
// we assume small number of files. spawnSync will raise a ENOBUFS error after maxBuffer
|
||||
for (const lp of dataLayout.localPaths()) {
|
||||
const [emptyDirsError, emptyDirs] = await safe(shell.spawn('find', [lp, '-type', 'd', '-empty'], { encoding: 'utf8', maxLines: 80000 }));
|
||||
if (emptyDirsError && emptyDirsError.stdoutLineCount >= 80000) throw new BoxError(BoxError.FS_ERROR, `Too many empty directories. Run "find ${lp} -type d -empty" to investigate`);
|
||||
if (emptyDirsError) throw emptyDirsError;
|
||||
if (emptyDirs.length) metadata.emptyDirs = metadata.emptyDirs.concat(emptyDirs.trim().split('\n').map((ed) => dataLayout.toRemotePath(ed)));
|
||||
|
||||
const [execFilesError, execFiles] = await safe(shell.spawn('find', [lp, '-type', 'f', '-executable'], { encoding: 'utf8', maxLines: 20000 }));
|
||||
if (execFilesError && execFilesError.stdoutLineCount >= 20000) throw new BoxError(BoxError.FS_ERROR, `Too many executable files. Run "find ${lp} -type f -executable" to investigate`);
|
||||
if (execFilesError) throw execFilesError;
|
||||
if (execFiles.length) metadata.execFiles = metadata.execFiles.concat(execFiles.trim().split('\n').map((ef) => dataLayout.toRemotePath(ef)));
|
||||
|
||||
const [symlinkFilesError, symlinkFiles] = await safe(shell.spawn('find', [lp, '-type', 'l'], { encoding: 'utf8', maxLines: 20000 }));
|
||||
if (symlinkFilesError && symlinkFilesError.stdoutLineCount >= 20000) throw new BoxError(BoxError.FS_ERROR, `Too many symlinks. Run "find ${lp} -type l" to investigate`);
|
||||
if (symlinkFilesError) throw symlinkFilesError;
|
||||
|
||||
if (symlinkFiles.length) metadata.symlinks = metadata.symlinks.concat(symlinkFiles.trim().split('\n').map((sl) => {
|
||||
const site = safe.fs.readlinkSync(sl);
|
||||
return { path: dataLayout.toRemotePath(sl), site };
|
||||
}));
|
||||
}
|
||||
|
||||
if (!safe.fs.writeFileSync(metadataFile, JSON.stringify(metadata, null, 2))) throw new BoxError(BoxError.FS_ERROR, `Error writing fs metadata: ${safe.error.message}`);
|
||||
}
|
||||
|
||||
async function restoreFsMetadata(dataLayout, metadataFile) {
|
||||
assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
|
||||
assert.strictEqual(typeof metadataFile, 'string');
|
||||
|
||||
debug(`Recreating empty directories in ${dataLayout.toString()}`);
|
||||
|
||||
const metadataJson = safe.fs.readFileSync(metadataFile, 'utf8');
|
||||
if (metadataJson === null) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Error loading fsmetadata.json:' + safe.error.message);
|
||||
const metadata = safe.JSON.parse(metadataJson);
|
||||
if (metadata === null) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Error parsing fsmetadata.json:' + safe.error.message);
|
||||
|
||||
for (const emptyDir of metadata.emptyDirs) {
|
||||
const [mkdirError] = await safe(fs.promises.mkdir(dataLayout.toLocalPath(emptyDir), { recursive: true }));
|
||||
if (mkdirError) throw new BoxError(BoxError.FS_ERROR, `unable to create path: ${mkdirError.message}`);
|
||||
}
|
||||
|
||||
for (const execFile of metadata.execFiles) {
|
||||
const [chmodError] = await safe(fs.promises.chmod(dataLayout.toLocalPath(execFile), parseInt('0755', 8)));
|
||||
if (chmodError) throw new BoxError(BoxError.FS_ERROR, `unable to chmod: ${chmodError.message}`);
|
||||
}
|
||||
|
||||
for (const symlink of (metadata.symlinks || [])) {
|
||||
if (!symlink.site) continue;
|
||||
// the path may not exist if we had a directory full of symlinks
|
||||
const [mkdirError] = await safe(fs.promises.mkdir(path.dirname(dataLayout.toLocalPath(symlink.path)), { recursive: true }));
|
||||
if (mkdirError) throw new BoxError(BoxError.FS_ERROR, `unable to symlink (mkdir): ${mkdirError.message}`);
|
||||
const [symlinkError] = await safe(fs.promises.symlink(symlink.site, dataLayout.toLocalPath(symlink.path), 'file'));
|
||||
if (symlinkError) throw new BoxError(BoxError.FS_ERROR, `unable to symlink: ${symlinkError.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function downloadDir(backupSite, remotePath, dataLayout, progressCallback) {
|
||||
assert.strictEqual(typeof backupSite, 'object');
|
||||
assert.strictEqual(typeof remotePath, 'string');
|
||||
@@ -297,21 +308,6 @@ async function upload(backupSite, remotePath, dataLayout, progressCallback) {
|
||||
return await sync(backupSite, remotePath, dataLayout, progressCallback); // { stats, integrityMap }
|
||||
}
|
||||
|
||||
async function copy(backupSite, fromPath, toPath, progressCallback) {
|
||||
assert.strictEqual(typeof backupSite, 'object');
|
||||
assert.strictEqual(typeof fromPath, 'string');
|
||||
assert.strictEqual(typeof toPath, 'string');
|
||||
assert.strictEqual(typeof progressCallback, 'function');
|
||||
|
||||
await backupSites.storageApi(backupSite).copyDir(backupSite.config, backupSite.limits, fromPath, toPath, progressCallback);
|
||||
}
|
||||
|
||||
function getFileExtension(encryption) {
|
||||
assert.strictEqual(typeof encryption, 'boolean');
|
||||
|
||||
return ''; // this also signals to backupcleanear that we are dealing with directories
|
||||
}
|
||||
|
||||
async function verify(backupSite, remotePath, integrityMap, progressCallback) {
|
||||
assert.strictEqual(typeof backupSite, 'object');
|
||||
assert.strictEqual(typeof remotePath, 'string');
|
||||
@@ -378,6 +374,6 @@ export default {
|
||||
verify,
|
||||
getFileExtension,
|
||||
copy,
|
||||
_saveFsMetadata,
|
||||
_restoreFsMetadata,
|
||||
_saveFsMetadata: saveFsMetadata,
|
||||
_restoreFsMetadata: restoreFsMetadata,
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user