storage: rework listDir api to be a generator (like) function

src/storage/filesystem.js

@@ -36,7 +36,6 @@ const assert = require('assert'),
     fs = require('fs'),
     path = require('path'),
     paths = require('../paths.js'),
-    { readdirp } = require('readdirp'),
     safe = require('safetydance'),
     shell = require('../shell.js')('filesystem');
 
@@ -114,34 +113,37 @@ async function exists(apiConfig, sourceFilePath) {
     return true;
 }
 
-function listDir(apiConfig, dir, batchSize, iteratorCallback, callback) {
+async function listDir(apiConfig, dir, batchSize, marker) {
     assert.strictEqual(typeof apiConfig, 'object');
     assert.strictEqual(typeof dir, 'string');
     assert.strictEqual(typeof batchSize, 'number');
-    assert.strictEqual(typeof iteratorCallback, 'function');
-    assert.strictEqual(typeof callback, 'function');
+    assert(typeof marker !== 'undefined');
 
-    let entries = [];
-    const entryStream = readdirp(dir, { type: 'files', alwaysStat: true, lstat: true });
-    entryStream.on('data', function (entryInfo) {
-        if (entryInfo.stats.isSymbolicLink()) return;
+    const stack = marker ? marker.stack : [dir];
+    const fileStream = marker ? marker.fileStream : [];
+    if (!marker) marker = { stack, fileStream };
 
-        entries.push({ fullPath: entryInfo.fullPath });
-        if (entries.length < batchSize) return;
-        entryStream.pause();
-        iteratorCallback(entries, function (error) {
-            if (error) return callback(error);
+    while (stack.length > 0) {
+        const currentDir = stack.pop();
+        const dirents = await fs.promises.readdir(currentDir, { withFileTypes: true });
 
-            entries = [];
-            entryStream.resume();
-        });
-    });
-    entryStream.on('warn', function (error) {
-        debug('listDir: warning. %o', error);
-    });
-    entryStream.on('end', function () {
-        iteratorCallback(entries, callback);
-    });
+        for (const dirent of dirents) {
+            const fullPath = path.join(currentDir, dirent.name);
+
+            if (dirent.isDirectory()) {
+                stack.push(fullPath);
+            } else if (dirent.isFile()) { // does not include symlink
+                const stat = await fs.promises.lstat(fullPath);
+                fileStream.push({ fullPath, size: stat.size });
+            }
+        }
+
+        if (fileStream.length >= batchSize) return { entries: fileStream.splice(0, batchSize), marker }; // note: splice also modifies the array
+    }
+
+    if (fileStream.length === 0) return { entries: [], marker: null };
+
+    return { entries: fileStream.splice(0, batchSize), marker };  // note: splice also modifies the array
 }
 
 async function copy(apiConfig, oldFilePath, newFilePath, progressCallback) {