backup: retry rsync file downloads

fixes #608

CHANGES

@@ -1518,4 +1518,5 @@
 * Improve various eventlog messages
 * Track dyndns change events
 * Add new S3 regions - Paris/Stockholm/Osaka
+* Retry errored downloads during restore
 

src/backups.js

@@ -508,18 +508,23 @@ function downloadDir(backupConfig, backupFilePath, destDir, progressCallback, ca
         mkdirp(path.dirname(destFilePath), function (error) {
             if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message));
 
-            api(backupConfig.provider).download(backupConfig, entry.fullPath, function (error, sourceStream) {
+            async.retry({ times: 5, interval: 20000 }, function (retryCallback) {
-                if (error) return callback(error);
-
-                sourceStream.on('error', callback);
-
                 let destStream = createWriteStream(destFilePath, backupConfig.key || null);
-                destStream.on('error', callback);
 
-                progressCallback({ message: `Downloading ${entry.fullPath} to ${destFilePath}` });
+                // protect against multiple errors. must destroy the write stream so that a previous retry does not write
+                let closeAndRetry = once((error) => { destStream.destroy(); retryCallback(error); });
 
-                sourceStream.pipe(destStream, { end: true }).on('finish', callback);
+                api(backupConfig.provider).download(backupConfig, entry.fullPath, function (error, sourceStream) {
-            });
+                    if (error) return closeAndRetry(error);
+
+                    sourceStream.on('error', closeAndRetry);
+                    destStream.on('error', closeAndRetry);
+
+                    progressCallback({ message: `Downloading ${entry.fullPath} to ${destFilePath}` });
+
+                    sourceStream.pipe(destStream, { end: true }).on('finish', closeAndRetry);
+                });
+            }, callback);
         });
     }