diff --git a/src/backupformat/rsync.js b/src/backupformat/rsync.js index ee670347f..d54bfc9f7 100644 --- a/src/backupformat/rsync.js +++ b/src/backupformat/rsync.js @@ -5,6 +5,7 @@ exports = module.exports = { upload, verify, getFileExtension, + copy, _saveFsMetadata: saveFsMetadata, _restoreFsMetadata: restoreFsMetadata @@ -292,6 +293,15 @@ async function upload(backupTarget, remotePath, dataLayout, progressCallback) { return await sync(backupTarget, remotePath, dataLayout, progressCallback); // { stats, integrityMap } } +async function copy(backupTarget, fromPath, toPath, progressCallback) { + assert.strictEqual(typeof backupTarget, 'object'); + assert.strictEqual(typeof fromPath, 'string'); + assert.strictEqual(typeof toPath, 'string'); + assert.strictEqual(typeof progressCallback, 'function'); + + await backupTargets.storageApi(backupTarget).copyDir(backupTarget.config, fromPath, toPath, progressCallback); +} + function getFileExtension(encryption) { assert.strictEqual(typeof encryption, 'boolean'); diff --git a/src/backupformat/tgz.js b/src/backupformat/tgz.js index db7f8f397..c9dd7fee9 100644 --- a/src/backupformat/tgz.js +++ b/src/backupformat/tgz.js @@ -271,6 +271,15 @@ async function upload(backupTarget, remotePath, dataLayout, progressCallback) { }); } +async function copy(backupTarget, fromPath, toPath, progressCallback) { + assert.strictEqual(typeof backupTarget, 'object'); + assert.strictEqual(typeof fromPath, 'string'); + assert.strictEqual(typeof toPath, 'string'); + assert.strictEqual(typeof progressCallback, 'function'); + + await backupTargets.storageApi(backupTarget).copy(backupTarget.config, fromPath, toPath, progressCallback); +} + async function verify(backupTarget, remotePath, integrityMap, progressCallback) { assert.strictEqual(typeof backupTarget, 'object'); assert.strictEqual(typeof remotePath, 'string'); @@ -340,6 +349,7 @@ exports = module.exports = { upload, verify, getFileExtension, + copy, // exported for testing _EnsureFileSizeStream: EnsureFileSizeStream diff --git a/src/backuptask.js b/src/backuptask.js index 29998f5a7..d9c88c32b 100644 --- a/src/backuptask.js +++ b/src/backuptask.js @@ -244,16 +244,16 @@ async function copy(backupTarget, srcRemotePath, destRemotePath, progressCallbac assert.strictEqual(typeof progressCallback, 'function'); const startTime = new Date(); - const [copyError] = await safe(backupTargets.storageApi(backupTarget).copy(backupTarget.config, srcRemotePath, destRemotePath, progressCallback)); + const [copyError] = await safe(backupFormats.api(backupTarget.format).copy(backupTarget, srcRemotePath, destRemotePath, progressCallback)); if (copyError) { - debug(`copy: copied to ${destRemotePath} errored. error: ${copyError.message}`); + debug(`copy: copy to ${destRemotePath} errored. error: ${copyError.message}`); throw copyError; } debug(`copy: copied successfully to ${destRemotePath}. Took ${(new Date() - startTime)/1000} seconds`); const [copyChecksumError] = await safe(backupTargets.storageApi(backupTarget).copy(backupTarget.config, `${srcRemotePath}.backupinfo`, `${destRemotePath}.backupinfo`, progressCallback)); if (copyChecksumError) { - debug(`copy: copied to ${destRemotePath} errored. error: ${copyChecksumError.message}`); + debug(`copy: copy to ${destRemotePath} errored. error: ${copyChecksumError.message}`); throw copyChecksumError; } debug(`copy: copied backupinfo successfully to ${destRemotePath}.backupinfo`); diff --git a/src/storage/filesystem.js b/src/storage/filesystem.js index 33ced5bd0..836888d7c 100644 --- a/src/storage/filesystem.js +++ b/src/storage/filesystem.js @@ -16,6 +16,7 @@ exports = module.exports = { download, copy, + copyDir, exists, listDir, @@ -183,10 +184,11 @@ async function listDir(config, remotePath, batchSize, marker) { return { entries: fileStream.splice(0, batchSize), marker }; // note: splice also modifies the array } -async function copy(config, fromPath, toPath, progressCallback) { +async function copyInternal(config, fromPath, toPath, options, progressCallback) { assert.strictEqual(typeof config, 'object'); assert.strictEqual(typeof fromPath, 'string'); assert.strictEqual(typeof toPath, 'string'); + assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof progressCallback, 'function'); const fullFromPath = path.join(getRootPath(config), fromPath); @@ -197,7 +199,8 @@ async function copy(config, fromPath, toPath, progressCallback) { progressCallback({ message: `Copying ${fullFromPath} to ${fullToPath}` }); - let cpOptions = ((config._provider !== mounts.MOUNT_TYPE_MOUNTPOINT && config._provider !== mounts.MOUNT_TYPE_CIFS) || config.preserveAttributes) ? '-a' : '-dR'; + let cpOptions = ((config._provider !== mounts.MOUNT_TYPE_MOUNTPOINT && config._provider !== mounts.MOUNT_TYPE_CIFS) || config.preserveAttributes) ? '-a' : '-d'; + if (options.recursive) cpOptions += 'R'; cpOptions += config.noHardlinks ? '' : 'l'; // this will hardlink backups saving space if (config._provider === mounts.MOUNT_TYPE_SSHFS) { @@ -215,6 +218,24 @@ async function copy(config, fromPath, toPath, progressCallback) { if (copyError) throw new BoxError(BoxError.EXTERNAL_ERROR, copyError.message); } +async function copy(config, fromPath, toPath, progressCallback) { + assert.strictEqual(typeof config, 'object'); + assert.strictEqual(typeof fromPath, 'string'); + assert.strictEqual(typeof toPath, 'string'); + assert.strictEqual(typeof progressCallback, 'function'); + + return await copyInternal(config, fromPath, toPath, { recursive: false }, progressCallback); +} + +async function copyDir(config, fromPath, toPath, progressCallback) { + assert.strictEqual(typeof config, 'object'); + assert.strictEqual(typeof fromPath, 'string'); + assert.strictEqual(typeof toPath, 'string'); + assert.strictEqual(typeof progressCallback, 'function'); + + return await copyInternal(config, fromPath, toPath, { recursive: true }, progressCallback); +} + async function remove(config, remotePath) { assert.strictEqual(typeof config, 'object'); assert.strictEqual(typeof remotePath, 'string'); diff --git a/src/storage/gcs.js b/src/storage/gcs.js index dd8f061cd..3e24d98a5 100644 --- a/src/storage/gcs.js +++ b/src/storage/gcs.js @@ -7,7 +7,9 @@ exports = module.exports = { upload, exists, download, + copy, + copyDir, listDir, @@ -131,7 +133,7 @@ async function listDir(apiConfig, remotePath, batchSize, marker) { const bucket = getBucket(apiConfig); const fullRemotePath = path.join(apiConfig.prefix, remotePath); - const query = marker || { prefix: fullRemotePath, autoPaginate: false, maxResults: batchSize }; + const query = marker || { prefix: fullRemotePath + '/', autoPaginate: false, maxResults: batchSize }; const [error, result] = await safe(bucket.getFiles(query)); if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, `Failed to get files: ${error.message}`); @@ -142,7 +144,7 @@ async function listDir(apiConfig, remotePath, batchSize, marker) { return { entries, marker: nextQuery || null }; } -async function copyFile(apiConfig, fullFromPath, fullToPath, progressCallback) { +async function copy(apiConfig, fullFromPath, fullToPath, progressCallback) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof fullFromPath, 'string'); assert.strictEqual(typeof fullToPath, 'string'); @@ -154,10 +156,11 @@ async function copyFile(apiConfig, fullFromPath, fullToPath, progressCallback) { if (copyError) throw new BoxError(BoxError.EXTERNAL_ERROR, copyError.message); } -async function copy(apiConfig, fromPath, toPath, progressCallback) { +async function copyDir(apiConfig, fromPath, toPath, options, progressCallback) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof fromPath, 'string'); assert.strictEqual(typeof toPath, 'string'); + assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof progressCallback, 'function'); const batchSize = 1000; @@ -173,7 +176,7 @@ async function copy(apiConfig, fromPath, toPath, progressCallback) { await async.eachLimit(batch.entries, concurrency, async (entry) => { const fullFromPath = path.join(apiConfig.prefix, entry.path); const fullToPath = path.join(apiConfig.prefix, toPath, path.relative(fromPath, entry.path)); - await copyFile(apiConfig, fullFromPath, fullToPath, progressCallback); + await copy(apiConfig, fullFromPath, fullToPath, progressCallback); }); if (!batch.marker) break; marker = batch.marker; diff --git a/src/storage/interface.js b/src/storage/interface.js index 934cd9092..7153c7478 100644 --- a/src/storage/interface.js +++ b/src/storage/interface.js @@ -20,6 +20,7 @@ exports = module.exports = { download, copy, + copyDir, listDir, @@ -94,6 +95,15 @@ async function copy(apiConfig, oldFilePath, newFilePath, progressCallback) { throw new BoxError(BoxError.NOT_IMPLEMENTED, 'copy is not implemented'); } +async function copyDir(apiConfig, oldFilePath, newFilePath, progressCallback) { + assert.strictEqual(typeof apiConfig, 'object'); + assert.strictEqual(typeof oldFilePath, 'string'); + assert.strictEqual(typeof newFilePath, 'string'); + assert.strictEqual(typeof progressCallback, 'function'); + + throw new BoxError(BoxError.NOT_IMPLEMENTED, 'copy is not implemented'); +} + async function listDir(apiConfig, dir, batchSize, marker) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof dir, 'string'); diff --git a/src/storage/s3.js b/src/storage/s3.js index 8358d8584..dcf1ad362 100644 --- a/src/storage/s3.js +++ b/src/storage/s3.js @@ -16,6 +16,7 @@ exports = module.exports = { exists, download, copy, + copyDir, listDir, @@ -376,7 +377,7 @@ function encodeCopySource(bucket, path) { return `/${bucket}/${output}`; } -async function copyFile(apiConfig, fullFromPath, fullToPath, fileSize, progressCallback) { +async function copyInternal(apiConfig, fullFromPath, fullToPath, fileSize, progressCallback) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof fullFromPath, 'string'); assert.strictEqual(typeof fullToPath, 'string'); @@ -477,19 +478,40 @@ async function copy(apiConfig, fromPath, toPath, progressCallback) { assert.strictEqual(typeof toPath, 'string'); assert.strictEqual(typeof progressCallback, 'function'); + const fullFromPath = path.join(apiConfig.prefix, fromPath); + const fullToPath = path.join(apiConfig.prefix, toPath); + + const params = { + Bucket: apiConfig.bucket, + Key: fullFromPath + }; + + const s3 = createS3Client(apiConfig, { retryStrategy: RETRY_STRATEGY }); // https://docs.aws.amazon.com/sdkref/latest/guide/feature-retry-behavior.html + const [error, data] = await safe(s3.headObject(params)); + if (error && S3_NOT_FOUND(error)) throw new BoxError(BoxError.NOT_FOUND, `Path ${fromPath} not found`); + if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, `Error headObject ${fromPath}. ${formatError(error)}`); + return await copyInternal(apiConfig, fullFromPath, fullToPath, data.ContentLength, progressCallback); +} + +async function copyDir(apiConfig, fromPath, toPath, progressCallback) { + assert.strictEqual(typeof apiConfig, 'object'); + assert.strictEqual(typeof fromPath, 'string'); + assert.strictEqual(typeof toPath, 'string'); + assert.strictEqual(typeof progressCallback, 'function'); + let total = 0; const concurrency = apiConfig.limits?.copyConcurrency || (apiConfig._provider === 's3' ? 500 : 10); - progressCallback({ message: `Copying with concurrency of ${concurrency}` }); + progressCallback({ message: `Copying ${fromPath} to ${toPath} with concurrency of ${concurrency}` }); let marker = null; while (true) { - const batch = await listDir(apiConfig, fromPath, 1000, marker); // returned entries are relative to fromPath + const batch = await listDir(apiConfig, fromPath, 1000, marker); // returned entries are relative to prefix total += batch.entries.length; progressCallback({ message: `Copying files from ${total-batch.entries.length}-${total}` }); await async.eachLimit(batch.entries, concurrency, async (entry) => { const fullFromPath = path.join(apiConfig.prefix, entry.path); const fullToPath = path.join(apiConfig.prefix, toPath, path.relative(fromPath, entry.path)); - await copyFile(apiConfig, fullFromPath, fullToPath, entry.size, progressCallback); + await copyInternal(apiConfig, fullFromPath, fullToPath, entry.size, progressCallback); }); if (!batch.marker) break; marker = batch.marker; @@ -543,7 +565,7 @@ async function removeDir(apiConfig, remotePathPrefix, progressCallback) { let total = 0; let marker = null; while (true) { - const batch = await listDir(apiConfig, remotePathPrefix, 1000, marker); // returns entries relative to remotePathPrefix + const batch = await listDir(apiConfig, remotePathPrefix, 1000, marker); // returns entries relative to (root) prefix const entries = batch.entries; total += entries.length;