diff --git a/src/storage/filesystem.js b/src/storage/filesystem.js index cd15183be..33ced5bd0 100644 --- a/src/storage/filesystem.js +++ b/src/storage/filesystem.js @@ -154,7 +154,8 @@ async function listDir(config, remotePath, batchSize, marker) { assert.strictEqual(typeof batchSize, 'number'); assert(typeof marker !== 'undefined'); - const fullRemotePath = path.join(getRootPath(config), remotePath); + const rootPath = getRootPath(config); + const fullRemotePath = path.join(rootPath, remotePath); const stack = marker ? marker.stack : [fullRemotePath]; const fileStream = marker ? marker.fileStream : []; if (!marker) marker = { stack, fileStream }; @@ -170,7 +171,7 @@ async function listDir(config, remotePath, batchSize, marker) { stack.push(fullEntryPath); } else if (dirent.isFile()) { // does not include symlink const stat = await fs.promises.lstat(fullEntryPath); - fileStream.push({ path: path.relative(fullRemotePath, fullEntryPath), size: stat.size }); + fileStream.push({ path: path.relative(rootPath, fullEntryPath), size: stat.size }); } } diff --git a/src/storage/gcs.js b/src/storage/gcs.js index b894964bb..dd8f061cd 100644 --- a/src/storage/gcs.js +++ b/src/storage/gcs.js @@ -138,7 +138,7 @@ async function listDir(apiConfig, remotePath, batchSize, marker) { const [files, nextQuery] = result; if (files.length === 0) return { entries: [], marker: null }; // no more - const entries = files.map(function (f) { return { path: path.relative(fullRemotePath, f.name) }; }); + const entries = files.map(function (f) { return { path: path.relative(apiConfig.prefix, f.name) }; }); return { entries, marker: nextQuery || null }; } @@ -171,8 +171,8 @@ async function copy(apiConfig, fromPath, toPath, progressCallback) { total += batch.entries.length; progressCallback({ message: `Copying ${batch.entries.length} files from ${batch.entries[0].path} to ${batch.entries[batch.entries.length-1].path}. total: ${total}` }); await async.eachLimit(batch.entries, concurrency, async (entry) => { - const fullFromPath = path.join(apiConfig.prefix, fromPath, entry.path); - const fullToPath = path.join(apiConfig.prefix, toPath, entry.path); + const fullFromPath = path.join(apiConfig.prefix, entry.path); + const fullToPath = path.join(apiConfig.prefix, toPath, path.relative(fromPath, entry.path)); await copyFile(apiConfig, fullFromPath, fullToPath, progressCallback); }); if (!batch.marker) break; diff --git a/src/storage/interface.js b/src/storage/interface.js index fa32b560a..934cd9092 100644 --- a/src/storage/interface.js +++ b/src/storage/interface.js @@ -101,7 +101,7 @@ async function listDir(apiConfig, dir, batchSize, marker) { assert(typeof marker !== 'undefined'); // Result: array of { path, size } - // path is relative to the dir being listed + // path is relative to the prefix/root and not the _dir_ being listed throw new BoxError(BoxError.NOT_IMPLEMENTED, 'listDir is not implemented'); } diff --git a/src/storage/s3.js b/src/storage/s3.js index 332a7b4a6..251126e90 100644 --- a/src/storage/s3.js +++ b/src/storage/s3.js @@ -359,7 +359,7 @@ async function listDir(apiConfig, remotePath, batchSize, marker) { const [error, listData] = await safe(s3.listObjectsV2(listParams)); if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, `Error listing objects in ${fullRemotePath}. ${formatError(error)}`); if (listData.KeyCount === 0 || listData.Contents.length === 0) return { entries: [], marker: null }; // no more - const entries = listData.Contents.map(function (c) { return { path: path.relative(fullRemotePath, c.Key), size: c.Size }; }); + const entries = listData.Contents.map(function (c) { return { path: path.relative(apiConfig.prefix, c.Key), size: c.Size }; }); return { entries, marker: !listData.IsTruncated ? null : listData.NextContinuationToken }; } @@ -487,8 +487,8 @@ async function copy(apiConfig, fromPath, toPath, progressCallback) { total += batch.entries.length; progressCallback({ message: `Copying files from ${total-batch.entries.length}-${total}` }); await async.eachLimit(batch.entries, concurrency, async (entry) => { - const fullFromPath = path.join(apiConfig.prefix, fromPath, entry.path); - const fullToPath = path.join(apiConfig.prefix, toPath, entry.path); + const fullFromPath = path.join(apiConfig.prefix, entry.path); + const fullToPath = path.join(apiConfig.prefix, toPath, path.relative(fromPath, entry.path)); await copyFile(apiConfig, fullFromPath, fullToPath, entry.size, progressCallback); }); if (!batch.marker) break; diff --git a/src/test/storage-provider-test.js b/src/test/storage-provider-test.js index 14af748fe..24c182acd 100644 --- a/src/test/storage-provider-test.js +++ b/src/test/storage-provider-test.js @@ -122,7 +122,7 @@ describe('Storage', function () { marker = result.marker; } - const expectedFiles = execSync(`find . -type f -printf '%P\n'`, { cwd: sourceDir, encoding: 'utf8' }).trim().split('\n'); + const expectedFiles = execSync(`find . -type f -printf '%P\n'`, { cwd: sourceDir, encoding: 'utf8' }).trim().split('\n').map(p => `storage/${p}`); expect(allFiles.map(function (f) { return f.path; }).sort()).to.eql(expectedFiles.sort()); }); @@ -189,7 +189,7 @@ describe('Storage', function () { const backupConfig = { provider: 's3', key: 'key', - prefix: 'unit.test', + prefix: 'prefix-test', bucket: 'cloudron-storage-test', accessKeyId: 'testkeyid', secretAccessKey: 'testsecret', @@ -239,7 +239,7 @@ describe('Storage', function () { async copyObject(params) { // CopySource already has the bucket path! // Key already has prefix but no bucket ptah! - // console.log('Copying:', path.join(basePath, params.CopySource), path.join(bucketPathNoPrefix, params.Key)); + console.log('Copying:', path.join(basePath, params.CopySource), path.join(bucketPathNoPrefix, params.Key)); await fs.promises.mkdir(path.dirname(path.join(bucketPathNoPrefix, params.Key)), { recursive: true }); await fs.promises.copyFile(path.join(basePath, params.CopySource.replace(/%2B/g, '+')), path.join(bucketPathNoPrefix, params.Key)); }