diff --git a/src/backupformat/rsync.js b/src/backupformat/rsync.js index d93305893..c8e20c352 100644 --- a/src/backupformat/rsync.js +++ b/src/backupformat/rsync.js @@ -194,7 +194,7 @@ async function downloadDir(backupTarget, backupFilePath, dataLayout, progressCal debug(`downloadDir: ${backupFilePath} to ${dataLayout.toString()}. encryption filenames: ${encryptedFilenames} content: ${!!backupTarget.encryption}`); async function downloadFile(entry) { - let relativePath = path.relative(backupFilePath, entry.fullPath); + let relativePath = path.relative(backupFilePath, entry.path); if (encryptedFilenames) { const { error, result } = hush.decryptFilePath(relativePath, backupTarget.encryption); if (error) throw new BoxError(BoxError.CRYPTO_ERROR, 'Unable to decrypt file'); @@ -206,17 +206,17 @@ async function downloadDir(backupTarget, backupFilePath, dataLayout, progressCal if (mkdirError) throw new BoxError(BoxError.FS_ERROR, mkdirError.message); await promiseRetry({ times: 3, interval: 20000 }, async function () { - const [downloadError, sourceStream] = await safe(backupTargets.storageApi(backupTarget).download(backupTarget.config, entry.fullPath)); + const [downloadError, sourceStream] = await safe(backupTargets.storageApi(backupTarget).download(backupTarget.config, entry.path)); if (downloadError) { - progressCallback({ message: `Download ${entry.fullPath} to ${destFilePath} errored: ${downloadError.message}` }); + progressCallback({ message: `Download ${entry.path} to ${destFilePath} errored: ${downloadError.message}` }); throw downloadError; } const ps = new ProgressStream({ interval: 10000 }); // display a progress every 10 seconds ps.on('progress', function (progress) { const transferred = Math.round(progress.transferred/1024/1024), speed = Math.round(progress.speed/1024/1024); - if (!transferred && !speed) return progressCallback({ message: `Downloading ${entry.fullPath}` }); // 0M@0MBps looks wrong - progressCallback({ message: `Downloading ${entry.fullPath}: ${transferred}M@${speed}MBps` }); + if (!transferred && !speed) return progressCallback({ message: `Downloading ${entry.path}` }); // 0M@0MBps looks wrong + progressCallback({ message: `Downloading ${entry.path}: ${transferred}M@${speed}MBps` }); }); const destStream = fs.createWriteStream(destFilePath); @@ -230,14 +230,14 @@ async function downloadDir(backupTarget, backupFilePath, dataLayout, progressCal streams.push(destStream); - progressCallback({ message: `Downloading ${entry.fullPath} to ${destFilePath}` }); + progressCallback({ message: `Downloading ${entry.path} to ${destFilePath}` }); const [pipelineError] = await safe(stream.pipeline(streams)); if (pipelineError) { - progressCallback({ message: `Download error ${entry.fullPath} to ${destFilePath}: ${pipelineError.message}` }); + progressCallback({ message: `Download error ${entry.path} to ${destFilePath}: ${pipelineError.message}` }); throw pipelineError; } - progressCallback({ message: `Download finished ${entry.fullPath} to ${destFilePath}` }); + progressCallback({ message: `Download finished ${entry.path} to ${destFilePath}` }); }); } diff --git a/src/storage/filesystem.js b/src/storage/filesystem.js index 97ce8cf00..fdda7856e 100644 --- a/src/storage/filesystem.js +++ b/src/storage/filesystem.js @@ -154,7 +154,7 @@ async function listDir(config, remotePath, batchSize, marker) { stack.push(fullEntryPath); } else if (dirent.isFile()) { // does not include symlink const stat = await fs.promises.lstat(fullEntryPath); - fileStream.push({ fullPath: path.relative(fullRemotePath, fullEntryPath), size: stat.size }); + fileStream.push({ path: path.relative(fullRemotePath, fullEntryPath), size: stat.size }); } } diff --git a/src/storage/gcs.js b/src/storage/gcs.js index 025b2aa38..462be9b6c 100644 --- a/src/storage/gcs.js +++ b/src/storage/gcs.js @@ -127,7 +127,7 @@ async function listDir(apiConfig, remotePath, batchSize, marker) { const [files, nextQuery] = result; if (files.length === 0) return { entries: [], marker: null }; // no more - const entries = files.map(function (f) { return { fullPath: path.relative(fullRemotePath, f.name) }; }); + const entries = files.map(function (f) { return { path: path.relative(fullRemotePath, f.name) }; }); return { entries, marker: nextQuery || null }; } @@ -158,10 +158,10 @@ async function copy(apiConfig, fromPath, toPath, progressCallback) { const batch = await listDir(apiConfig, fromPath, batchSize, marker); // returns entries relative to fromPath if (batch.entries.length === 0) break; total += batch.entries.length; - progressCallback({ message: `Copying ${batch.entries.length} files from ${batch.entries[0].fullPath} to ${batch.entries[batch.entries.length-1].fullPath}. total: ${total}` }); + progressCallback({ message: `Copying ${batch.entries.length} files from ${batch.entries[0].path} to ${batch.entries[batch.entries.length-1].path}. total: ${total}` }); await async.eachLimit(batch.entries, concurrency, async (entry) => { - const fullFromPath = path.join(apiConfig.prefix, fromPath, entry.fullPath); - const fullToPath = path.join(apiConfig.prefix, toPath, entry.fullPath); + const fullFromPath = path.join(apiConfig.prefix, fromPath, entry.path); + const fullToPath = path.join(apiConfig.prefix, toPath, entry.path); await copyFile(apiConfig, fullFromPath, fullToPath, progressCallback); }); if (!batch.marker) break; @@ -193,8 +193,8 @@ async function removeDir(apiConfig, remotePathPrefix, progressCallback) { if (batch.entries.length === 0) break; const entries = batch.entries; total += entries.length; - progressCallback({ message: `Removing ${entries.length} files from ${entries[0].fullPath} to ${entries[entries.length-1].fullPath}. total: ${total}` }); - await async.eachLimit(entries, concurrency, async (entry) => await remove(apiConfig, entry.fullPath)); // remove will add 'prefix' + progressCallback({ message: `Removing ${entries.length} files from ${entries[0].path} to ${entries[entries.length-1].path}. total: ${total}` }); + await async.eachLimit(entries, concurrency, async (entry) => await remove(apiConfig, entry.path)); // remove will add 'prefix' if (!batch.marker) break; marker = batch.marker; } diff --git a/src/storage/interface.js b/src/storage/interface.js index 41bac0d95..83de18a2a 100644 --- a/src/storage/interface.js +++ b/src/storage/interface.js @@ -92,8 +92,8 @@ async function listDir(apiConfig, dir, batchSize, marker) { assert.strictEqual(typeof batchSize, 'number'); assert(typeof marker !== 'undefined'); - // Result: array of { fullPath, size } - // fullPath is relative to the dir being listed + // Result: array of { path, size } + // path is relative to the dir being listed throw new BoxError(BoxError.NOT_IMPLEMENTED, 'listDir is not implemented'); } diff --git a/src/storage/s3.js b/src/storage/s3.js index ccb03ce49..f15bdb960 100644 --- a/src/storage/s3.js +++ b/src/storage/s3.js @@ -341,7 +341,7 @@ async function listDir(apiConfig, remotePath, batchSize, marker) { const [error, listData] = await safe(s3.listObjectsV2(listParams)); if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, `Error listing objects in ${fullRemotePath}. ${formatError(error)}`); if (listData.KeyCount === 0 || listData.Contents.length === 0) return { entries: [], marker: null }; // no more - const entries = listData.Contents.map(function (c) { return { fullPath: path.relative(fullRemotePath, c.Key), size: c.Size }; }); + const entries = listData.Contents.map(function (c) { return { path: path.relative(fullRemotePath, c.Key), size: c.Size }; }); return { entries, marker: !listData.IsTruncated ? null : listData.NextContinuationToken }; } @@ -469,8 +469,8 @@ async function copy(apiConfig, fromPath, toPath, progressCallback) { total += batch.entries.length; progressCallback({ message: `Copying files from ${total-batch.entries.length}-${total}` }); await async.eachLimit(batch.entries, concurrency, async (entry) => { - const fullFromPath = path.join(apiConfig.prefix, fromPath, entry.fullPath); - const fullToPath = path.join(apiConfig.prefix, toPath, entry.fullPath); + const fullFromPath = path.join(apiConfig.prefix, fromPath, entry.path); + const fullToPath = path.join(apiConfig.prefix, toPath, entry.path); await copyFile(apiConfig, fullFromPath, fullToPath, entry.size, progressCallback); }); if (!batch.marker) break; @@ -537,11 +537,11 @@ async function removeDir(apiConfig, remotePathPrefix, progressCallback) { const deleteParams = { Bucket: apiConfig.bucket, Delete: { - Objects: objects.map(function (o) { return { Key: path.join(apiConfig.prefix, o.fullPath) }; }) + Objects: objects.map(function (o) { return { Key: path.join(apiConfig.prefix, o.path) }; }) } }; - const fullFirstPath = path.join(apiConfig.prefix, objects[0].fullPath), fullLastPath = path.join(apiConfig.prefix, objects[objects.length-1].fullPath); + const fullFirstPath = path.join(apiConfig.prefix, objects[0].path), fullLastPath = path.join(apiConfig.prefix, objects[objects.length-1].path); progressCallback({ message: `Removing ${objects.length} files from ${fullFirstPath} to ${fullLastPath}` }); // deleteObjects does not return error if key is not found diff --git a/src/test/storage-provider-test.js b/src/test/storage-provider-test.js index b1d5fedbb..7484b6e84 100644 --- a/src/test/storage-provider-test.js +++ b/src/test/storage-provider-test.js @@ -123,7 +123,7 @@ describe('Storage', function () { } const expectedFiles = execSync(`find . -type f -printf '%P\n'`, { cwd: sourceDir, encoding: 'utf8' }).trim().split('\n'); - expect(allFiles.map(function (f) { return f.fullPath; }).sort()).to.eql(expectedFiles.sort()); + expect(allFiles.map(function (f) { return f.path; }).sort()).to.eql(expectedFiles.sort()); }); it('can copy', async function () { @@ -295,7 +295,7 @@ describe('Storage', function () { marker = result.marker; } - expect(allFiles.map(function (f) { return f.fullPath; })).to.contain('uploadtest/test.txt'); + expect(allFiles.map(function (f) { return f.path; })).to.contain('uploadtest/test.txt'); }); it('can copy', async function () { @@ -344,7 +344,6 @@ describe('Storage', function () { function getFullWritablePath(key) { const fullPath = path.join(bucketPathNoPrefix, key); fs.mkdirSync(path.dirname(fullPath), { recursive: true }); - console.log(fullPath); return fullPath; } @@ -433,7 +432,7 @@ describe('Storage', function () { marker = result.marker; } - expect(allFiles.map(function (f) { return f.fullPath; })).to.contain('uploadtest/test.txt'); + expect(allFiles.map(function (f) { return f.path; })).to.contain('uploadtest/test.txt'); }); it('can copy', async function () {