storage: add copyDir

we changed listDir in c44863a9bb to list
a directory . this broke copy for files since a '/' is added when listing
the file.

src/storage/filesystem.js

@@ -16,6 +16,7 @@ exports = module.exports = {
     download,
 
     copy,
+    copyDir,
 
     exists,
     listDir,
@@ -183,10 +184,11 @@ async function listDir(config, remotePath, batchSize, marker) {
     return { entries: fileStream.splice(0, batchSize), marker };  // note: splice also modifies the array
 }
 
-async function copy(config, fromPath, toPath, progressCallback) {
+async function copyInternal(config, fromPath, toPath, options, progressCallback) {
     assert.strictEqual(typeof config, 'object');
     assert.strictEqual(typeof fromPath, 'string');
     assert.strictEqual(typeof toPath, 'string');
+    assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof progressCallback, 'function');
 
     const fullFromPath = path.join(getRootPath(config), fromPath);
@@ -197,7 +199,8 @@ async function copy(config, fromPath, toPath, progressCallback) {
 
     progressCallback({ message: `Copying ${fullFromPath} to ${fullToPath}` });
 
-    let cpOptions = ((config._provider !== mounts.MOUNT_TYPE_MOUNTPOINT && config._provider !== mounts.MOUNT_TYPE_CIFS) || config.preserveAttributes) ? '-a' : '-dR';
+    let cpOptions = ((config._provider !== mounts.MOUNT_TYPE_MOUNTPOINT && config._provider !== mounts.MOUNT_TYPE_CIFS) || config.preserveAttributes) ? '-a' : '-d';
+    if (options.recursive) cpOptions += 'R';
     cpOptions += config.noHardlinks ? '' : 'l';  // this will hardlink backups saving space
 
     if (config._provider === mounts.MOUNT_TYPE_SSHFS) {
@@ -215,6 +218,24 @@ async function copy(config, fromPath, toPath, progressCallback) {
     if (copyError) throw new BoxError(BoxError.EXTERNAL_ERROR, copyError.message);
 }
 
+async function copy(config, fromPath, toPath, progressCallback) {
+    assert.strictEqual(typeof config, 'object');
+    assert.strictEqual(typeof fromPath, 'string');
+    assert.strictEqual(typeof toPath, 'string');
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    return await copyInternal(config, fromPath, toPath, { recursive: false }, progressCallback);
+}
+
+async function copyDir(config, fromPath, toPath, progressCallback) {
+    assert.strictEqual(typeof config, 'object');
+    assert.strictEqual(typeof fromPath, 'string');
+    assert.strictEqual(typeof toPath, 'string');
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    return await copyInternal(config, fromPath, toPath, { recursive: true }, progressCallback);
+}
+
 async function remove(config, remotePath) {
     assert.strictEqual(typeof config, 'object');
     assert.strictEqual(typeof remotePath, 'string');

src/storage/gcs.js

@@ -7,7 +7,9 @@ exports = module.exports = {
     upload,
     exists,
     download,
+
     copy,
+    copyDir,
 
     listDir,
 
@@ -131,7 +133,7 @@ async function listDir(apiConfig, remotePath, batchSize, marker) {
     const bucket = getBucket(apiConfig);
     const fullRemotePath = path.join(apiConfig.prefix, remotePath);
 
-    const query = marker || { prefix: fullRemotePath, autoPaginate: false, maxResults: batchSize };
+    const query = marker || { prefix: fullRemotePath + '/', autoPaginate: false, maxResults: batchSize };
 
     const [error, result] = await safe(bucket.getFiles(query));
     if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, `Failed to get files: ${error.message}`);
@@ -142,7 +144,7 @@ async function listDir(apiConfig, remotePath, batchSize, marker) {
     return { entries, marker: nextQuery || null };
 }
 
-async function copyFile(apiConfig, fullFromPath, fullToPath, progressCallback) {
+async function copy(apiConfig, fullFromPath, fullToPath, progressCallback) {
     assert.strictEqual(typeof apiConfig, 'object');
     assert.strictEqual(typeof fullFromPath, 'string');
     assert.strictEqual(typeof fullToPath, 'string');
@@ -154,10 +156,11 @@ async function copyFile(apiConfig, fullFromPath, fullToPath, progressCallback) {
     if (copyError) throw new BoxError(BoxError.EXTERNAL_ERROR, copyError.message);
 }
 
-async function copy(apiConfig, fromPath, toPath, progressCallback) {
+async function copyDir(apiConfig, fromPath, toPath, options, progressCallback) {
     assert.strictEqual(typeof apiConfig, 'object');
     assert.strictEqual(typeof fromPath, 'string');
     assert.strictEqual(typeof toPath, 'string');
+    assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof progressCallback, 'function');
 
     const batchSize = 1000;
@@ -173,7 +176,7 @@ async function copy(apiConfig, fromPath, toPath, progressCallback) {
         await async.eachLimit(batch.entries, concurrency, async (entry) => {
             const fullFromPath = path.join(apiConfig.prefix, entry.path);
             const fullToPath = path.join(apiConfig.prefix, toPath, path.relative(fromPath, entry.path));
-            await copyFile(apiConfig, fullFromPath, fullToPath, progressCallback);
+            await copy(apiConfig, fullFromPath, fullToPath, progressCallback);
         });
         if (!batch.marker) break;
         marker = batch.marker;

src/storage/interface.js

@@ -20,6 +20,7 @@ exports = module.exports = {
 
     download,
     copy,
+    copyDir,
 
     listDir,
 
@@ -94,6 +95,15 @@ async function copy(apiConfig, oldFilePath, newFilePath, progressCallback) {
     throw new BoxError(BoxError.NOT_IMPLEMENTED, 'copy is not implemented');
 }
 
+async function copyDir(apiConfig, oldFilePath, newFilePath, progressCallback) {
+    assert.strictEqual(typeof apiConfig, 'object');
+    assert.strictEqual(typeof oldFilePath, 'string');
+    assert.strictEqual(typeof newFilePath, 'string');
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    throw new BoxError(BoxError.NOT_IMPLEMENTED, 'copy is not implemented');
+}
+
 async function listDir(apiConfig, dir, batchSize, marker) {
     assert.strictEqual(typeof apiConfig, 'object');
     assert.strictEqual(typeof dir, 'string');

src/storage/s3.js

@@ -16,6 +16,7 @@ exports = module.exports = {
     exists,
     download,
     copy,
+    copyDir,
 
     listDir,
 
@@ -376,7 +377,7 @@ function encodeCopySource(bucket, path) {
     return `/${bucket}/${output}`;
 }
 
-async function copyFile(apiConfig, fullFromPath, fullToPath, fileSize, progressCallback) {
+async function copyInternal(apiConfig, fullFromPath, fullToPath, fileSize, progressCallback) {
     assert.strictEqual(typeof apiConfig, 'object');
     assert.strictEqual(typeof fullFromPath, 'string');
     assert.strictEqual(typeof fullToPath, 'string');
@@ -477,19 +478,40 @@ async function copy(apiConfig, fromPath, toPath, progressCallback) {
     assert.strictEqual(typeof toPath, 'string');
     assert.strictEqual(typeof progressCallback, 'function');
 
+    const fullFromPath = path.join(apiConfig.prefix, fromPath);
+    const fullToPath = path.join(apiConfig.prefix, toPath);
+
+    const params = {
+        Bucket: apiConfig.bucket,
+        Key: fullFromPath
+    };
+
+    const s3 = createS3Client(apiConfig, { retryStrategy: RETRY_STRATEGY }); // https://docs.aws.amazon.com/sdkref/latest/guide/feature-retry-behavior.html
+    const [error, data] = await safe(s3.headObject(params));
+    if (error && S3_NOT_FOUND(error)) throw new BoxError(BoxError.NOT_FOUND, `Path ${fromPath} not found`);
+    if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, `Error headObject ${fromPath}. ${formatError(error)}`);
+    return await copyInternal(apiConfig, fullFromPath, fullToPath, data.ContentLength, progressCallback);
+}
+
+async function copyDir(apiConfig, fromPath, toPath, progressCallback) {
+    assert.strictEqual(typeof apiConfig, 'object');
+    assert.strictEqual(typeof fromPath, 'string');
+    assert.strictEqual(typeof toPath, 'string');
+    assert.strictEqual(typeof progressCallback, 'function');
+
     let total = 0;
     const concurrency = apiConfig.limits?.copyConcurrency || (apiConfig._provider === 's3' ? 500 : 10);
-    progressCallback({ message: `Copying with concurrency of ${concurrency}` });
+    progressCallback({ message: `Copying ${fromPath} to ${toPath} with concurrency of ${concurrency}` });
 
     let marker = null;
     while (true) {
-        const batch = await listDir(apiConfig, fromPath, 1000, marker); // returned entries are relative to fromPath
+        const batch = await listDir(apiConfig, fromPath, 1000, marker); // returned entries are relative to prefix
         total += batch.entries.length;
         progressCallback({ message: `Copying files from ${total-batch.entries.length}-${total}` });
         await async.eachLimit(batch.entries, concurrency, async (entry) => {
             const fullFromPath = path.join(apiConfig.prefix, entry.path);
             const fullToPath = path.join(apiConfig.prefix, toPath, path.relative(fromPath, entry.path));
-            await copyFile(apiConfig, fullFromPath, fullToPath, entry.size, progressCallback);
+            await copyInternal(apiConfig, fullFromPath, fullToPath, entry.size, progressCallback);
         });
         if (!batch.marker) break;
         marker = batch.marker;
@@ -543,7 +565,7 @@ async function removeDir(apiConfig, remotePathPrefix, progressCallback) {
     let total = 0;
     let marker = null;
     while (true) {
-        const batch = await listDir(apiConfig, remotePathPrefix, 1000, marker); // returns entries relative to remotePathPrefix
+        const batch = await listDir(apiConfig, remotePathPrefix, 1000, marker); // returns entries relative to (root) prefix
 
         const entries = batch.entries;
         total += entries.length;