diff --git a/CHANGES b/CHANGES index 4c91fe479..484927d81 100644 --- a/CHANGES +++ b/CHANGES @@ -1518,4 +1518,5 @@ * Improve various eventlog messages * Track dyndns change events * Add new S3 regions - Paris/Stockholm/Osaka +* Retry errored downloads during restore diff --git a/src/backups.js b/src/backups.js index b9b3cebfb..b80be7435 100644 --- a/src/backups.js +++ b/src/backups.js @@ -508,18 +508,23 @@ function downloadDir(backupConfig, backupFilePath, destDir, progressCallback, ca mkdirp(path.dirname(destFilePath), function (error) { if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message)); - api(backupConfig.provider).download(backupConfig, entry.fullPath, function (error, sourceStream) { - if (error) return callback(error); - - sourceStream.on('error', callback); - + async.retry({ times: 5, interval: 20000 }, function (retryCallback) { let destStream = createWriteStream(destFilePath, backupConfig.key || null); - destStream.on('error', callback); - progressCallback({ message: `Downloading ${entry.fullPath} to ${destFilePath}` }); + // protect against multiple errors. must destroy the write stream so that a previous retry does not write + let closeAndRetry = once((error) => { destStream.destroy(); retryCallback(error); }); - sourceStream.pipe(destStream, { end: true }).on('finish', callback); - }); + api(backupConfig.provider).download(backupConfig, entry.fullPath, function (error, sourceStream) { + if (error) return closeAndRetry(error); + + sourceStream.on('error', closeAndRetry); + destStream.on('error', closeAndRetry); + + progressCallback({ message: `Downloading ${entry.fullPath} to ${destFilePath}` }); + + sourceStream.pipe(destStream, { end: true }).on('finish', closeAndRetry); + }); + }, callback); }); }