make syncer track directories

src/backups.js

@@ -238,6 +238,8 @@ function sync(backupConfig, backupId, dataDir, callback) {
         } else if (task.operation === 'removedir') {
             safe.fs.writeFileSync(paths.BACKUP_RESULT_FILE, 'Removing directory ' + task.path);
             api(backupConfig.provider).removeDir(backupConfig, backupFilePath, iteratorCallback);
+        } else if (task.operation === 'mkdir') { // unused
+            safe.fs.writeFileSync(paths.BACKUP_RESULT_FILE, 'Adding directory ' + task.path);
         }
     }, 10 /* concurrency */, function (error) {
         if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message));

src/storage/filesystem.js

@@ -20,7 +20,6 @@ var assert = require('assert'),
     debug = require('debug')('box:storage/filesystem'),
     fs = require('fs'),
     mkdirp = require('mkdirp'),
-    PassThrough = require('stream').PassThrough,
     path = require('path'),
     safe = require('safetydance'),
     shell = require('../shell.js');

src/syncer.js

@@ -3,6 +3,7 @@
 var assert = require('assert'),
     async = require('async'),
     debug = require('debug')('box:syncer'),
+    fs = require('fs'),
     path = require('path'),
     paths = require('./paths.js'),
     safe = require('safetydance');
@@ -29,6 +30,14 @@ function readTree(dir) {
     return list.map(function (e) { return { stat: safe.fs.lstatSync(path.join(dir, e)), name: e }; });
 }
 
+function ISDIR(x) {
+    return (x & fs.constants.S_IFDIR) === fs.constants.S_IFDIR;
+}
+
+function ISFILE(x) {
+    return (x & fs.constants.S_IFREG) === fs.constants.S_IFREG;
+}
+
 function sync(dir, taskProcessor, concurrency, callback) {
     assert.strictEqual(typeof dir, 'string');
     assert.strictEqual(typeof taskProcessor, 'function');
@@ -41,7 +50,7 @@ function sync(dir, taskProcessor, concurrency, callback) {
         newCacheFile = path.join(paths.BACKUP_INFO_DIR, path.basename(dir) + '.sync.cache.new');
 
     if (!safe.fs.existsSync(cacheFile)) { // if cache is missing, start out empty. TODO: do a remote listDir and rebuild
-        delQueue.push({ operation: 'removedir', path: '' });
+        delQueue.push({ operation: 'removedir', path: '', reason: 'nocache' });
     }
 
     var cache = readCache(cacheFile);
@@ -50,39 +59,65 @@ function sync(dir, taskProcessor, concurrency, callback) {
     if (newCacheFd === -1) return callback(new Error('Error opening new cache file: ' + safe.error.message));
 
     function advanceCache(entryPath) {
-        // TODO: detect and issue removedir
+        var lastRemovedDir = null;
+
         for (; curCacheIndex !== cache.length && (entryPath === '' || cache[curCacheIndex].path < entryPath); ++curCacheIndex) {
-            delQueue.push({ operation: 'remove', path: cache[curCacheIndex].path });
+            // ignore subdirs of lastRemovedDir since it was removed already
+            if (lastRemovedDir && cache[curCacheIndex].path.startsWith(lastRemovedDir)) continue;
+
+            if (ISDIR(cache[curCacheIndex].stat.mode)) {
+                delQueue.push({ operation: 'removedir', path: cache[curCacheIndex].path, reason: 'missing' });
+                lastRemovedDir = cache[curCacheIndex].path;
+            } else {
+                delQueue.push({ operation: 'remove', path: cache[curCacheIndex].path, reason: 'missing' });
+                lastRemovedDir = null;
+            }
         }
     }
 
     function traverse(relpath) {
         var entries = readTree(path.join(dir, relpath));
 
+        // addQueue.push({ operation: 'mkdir', path: relpath });
         for (var i = 0; i < entries.length; i++) {
             var entryPath = path.join(relpath, entries[i].name);
-            var stat = entries[i].stat;
+            var entryStat = entries[i].stat;
 
-            if (!stat) continue; // some stat error
-            if (!stat.isDirectory() && !stat.isFile()) continue;
-            if (stat.isSymbolicLink()) continue;
+            if (!entryStat) continue; // some stat error. prented it doesn't exist
+            if (!entryStat.isDirectory() && !entryStat.isFile()) continue; // ignore non-files and dirs
+            if (entryStat.isSymbolicLink()) continue;
 
-            if (stat.isDirectory()) {
-                traverse(entryPath);
-                continue;
+            safe.fs.appendFileSync(newCacheFd, JSON.stringify({ path: entryPath, stat: { mtime: entryStat.mtime.getTime(), size: entryStat.size, inode: entryStat.inode, mode: entryStat.mode } }) + '\n');
+
+            if (curCacheIndex !== cache.length && cache[curCacheIndex].path < entryPath) { // files disappeared. first advance cache as needed
+                advanceCache(entryPath);
             }
 
-            safe.fs.appendFileSync(newCacheFd, JSON.stringify({ path: entryPath, mtime: stat.mtime.getTime(), size: stat.size, inode: stat.inode  }) + '\n');
+            const cachePath = curCacheIndex === cache.length ? null : cache[curCacheIndex].path;
+            const cacheStat = curCacheIndex === cache.length ? null : cache[curCacheIndex].stat;
 
-            advanceCache(entryPath);
-
-            if (curCacheIndex !== cache.length && cache[curCacheIndex].path === entryPath) {
-                if (stat.mtime.getTime() !== cache[curCacheIndex].mtime || stat.size != cache[curCacheIndex].size || stat.inode !== cache[curCacheIndex].inode) {
-                    addQueue.push({ operation: 'add', path: entryPath });
+            if (cachePath === null || cachePath > entryPath) { // new files appeared
+                if (entryStat.isDirectory()) {
+                    traverse(entryPath);
+                } else {
+                    addQueue.push({ operation: 'add', path: entryPath, reason: 'new' });
+                }
+            } else if (ISDIR(cacheStat.mode) && entryStat.isDirectory()) { // dir names match
+                ++curCacheIndex;
+                traverse(entryPath);
+            } else if (ISFILE(cacheStat.mode) && entryStat.isFile()) { // file names match
+                if (entryStat.mtime.getTime() !== cacheStat.mtime || entryStat.size != cacheStat.size || entryStat.inode !== cacheStat.inode) { // file changed
+                    addQueue.push({ operation: 'add', path: entryPath, reason: 'changed' });
                 }
                 ++curCacheIndex;
-            } else {
-                addQueue.push({ operation: 'add', path: entryPath });
+            } else if (entryStat.isDirectory()) { // was a file, now a directory
+                delQueue.push({ operation: 'remove', path: cachePath, reason: 'wasfile' });
+                ++curCacheIndex;
+                traverse(entryPath);
+            } else { // was a dir, now a file
+                delQueue.push({ operation: 'removedir', path: cachePath, reason: 'wasdir' });
+                while (curCacheIndex !== cache.length && cache[curCacheIndex].path.startsWith(cachePath)) ++curCacheIndex;
+                addQueue.push({ operation: 'add', path: entryPath, reason: 'wasdir' });
             }
         }
     }

src/test/common.js

@@ -0,0 +1,33 @@
+'use strict';
+
+var fs = require('fs'),
+    mkdirp = require('mkdirp'),
+    path = require('path'),
+    rimraf = require('rimraf');
+
+exports = module.exports = {
+    createTree: createTree
+};
+
+function createTree(root, obj) {
+    rimraf.sync(root);
+    mkdirp.sync(root);
+
+    function createSubTree(tree, curpath) {
+        for (var key in tree) {
+            if (typeof tree[key] === 'string') {
+                if (key.startsWith('link:')) {
+                    fs.symlinkSync(tree[key], path.join(curpath, key.slice(5)));
+                } else {
+                    fs.writeFileSync(path.join(curpath, key), tree[key], 'utf8');
+                }
+            } else {
+                fs.mkdirSync(path.join(curpath, key));
+                createSubTree(tree[key], path.join(curpath, key));
+            }
+        }
+    }
+
+    createSubTree(obj, root);
+}
+

src/test/syncer-test.js

@@ -0,0 +1,330 @@
+/* global it:false */
+/* global describe:false */
+/* global before:false */
+/* global after:false */
+
+'use strict';
+
+var createTree = require('./common.js').createTree,
+    execSync = require('child_process').execSync,
+    expect = require('expect.js'),
+    fs = require('fs'),
+    os = require('os'),
+    path = require('path'),
+    paths = require('../paths.js'),
+    safe = require('safetydance'),
+    syncer = require('../syncer.js');
+
+var gTasks = [ ],
+    gTmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'syncer-test')),
+    gCacheFile = path.join(paths.BACKUP_INFO_DIR, path.basename(gTmpDir) + '.sync.cache');
+
+function collectTasks(task, callback) {
+    gTasks.push(task);
+    callback();
+}
+
+describe('Syncer', function () {
+    before(function () {
+        console.log('Tests are run in %s with cache file %s', gTmpDir, gCacheFile)
+    });
+
+    it('missing cache - removes remote dir', function (done) {
+        gTasks = [ ];
+        safe.fs.unlinkSync(gCacheFile);
+        createTree(gTmpDir, { });
+
+        syncer.sync(gTmpDir, collectTasks, 10, function (error) {
+            expect(error).to.not.be.ok();
+
+            expect(gTasks).to.eql([
+                { operation: 'removedir', path: '', reason: 'nocache' }
+            ]);
+            done();
+        });
+    });
+
+    it('empty cache - adds all', function (done) {
+        gTasks = [ ];
+        fs.writeFileSync(gCacheFile, '', 'utf8');
+        createTree(gTmpDir, { 'src': { 'index.js': 'some code' }, 'test': { 'test.js': 'This is a README' }, 'walrus': 'animal' });
+
+        syncer.sync(gTmpDir, collectTasks, 10, function (error) {
+            expect(error).to.not.be.ok();
+
+            expect(gTasks).to.eql([
+                { operation: 'add', path: 'src/index.js', reason: 'new' },
+                { operation: 'add', path: 'test/test.js', reason: 'new' },
+                { operation: 'add', path: 'walrus', reason: 'new' }
+            ]);
+            done();
+        });
+    });
+
+    it('empty cache - deep', function (done) {
+        gTasks = [ ];
+        fs.writeFileSync(gCacheFile, '', 'utf8');
+        createTree(gTmpDir, { a: { b: { c: { d: { e: 'some code' } } } } });
+
+        syncer.sync(gTmpDir, collectTasks, 10, function (error) {
+            expect(error).to.not.be.ok();
+
+            expect(gTasks).to.eql([
+                { operation: 'add', path: 'a/b/c/d/e', reason: 'new' }
+            ]);
+            done();
+        });
+    });
+
+    it('ignores special files', function (done) {
+        gTasks = [ ];
+        fs.writeFileSync(gCacheFile, '', 'utf8');
+        createTree(gTmpDir, { 'link:file': '/tmp', 'readme': 'this is readme' });
+
+        syncer.sync(gTmpDir, collectTasks, 10, function (error) {
+            expect(error).to.not.be.ok();
+
+            expect(gTasks).to.eql([
+                { operation: 'add', path: 'readme', reason: 'new' }
+            ]);
+            done();
+        });
+    });
+
+    it('adds changed files', function (done) {
+        gTasks = [ ];
+        fs.writeFileSync(gCacheFile, '', 'utf8');
+        createTree(gTmpDir, { 'src': { 'index.js': 'some code' }, 'test': { 'test.js': 'This is a README' }, 'walrus': 'animal' });
+
+        syncer.sync(gTmpDir, collectTasks, 10, function (error) {
+            expect(error).to.not.be.ok();
+            expect(gTasks.length).to.be(3);
+
+            execSync(`touch src/index.js test/test.js`, { cwd: gTmpDir });
+
+            gTasks = [ ];
+            syncer.sync(gTmpDir, collectTasks, 10, function (error) {
+                expect(error).to.not.be.ok();
+
+                expect(gTasks).to.eql([
+                    { operation: 'add', path: 'src/index.js', reason: 'changed' },
+                    { operation: 'add', path: 'test/test.js', reason: 'changed' }
+                ]);
+
+                done();
+            });
+        });
+    });
+
+    it('removes missing files', function (done) {
+        gTasks = [ ];
+        fs.writeFileSync(gCacheFile, '', 'utf8');
+        createTree(gTmpDir, { 'src': { 'index.js': 'some code' }, 'test': { 'test.js': 'This is a README' }, 'walrus': 'animal' });
+
+        syncer.sync(gTmpDir, collectTasks, 10, function (error) {
+            expect(error).to.not.be.ok();
+            expect(gTasks.length).to.be(3);
+
+            execSync(`rm src/index.js walrus`, { cwd: gTmpDir });
+
+            gTasks = [ ];
+            syncer.sync(gTmpDir, collectTasks, 10, function (error) {
+                expect(error).to.not.be.ok();
+
+                expect(gTasks).to.eql([
+                    { operation: 'remove', path: 'src/index.js', reason: 'missing' },
+                    { operation: 'remove', path: 'walrus', reason: 'missing' }
+                ]);
+
+                done();
+            });
+        });
+    });
+
+    it('removes missing dirs', function (done) {
+        gTasks = [ ];
+        fs.writeFileSync(gCacheFile, '', 'utf8');
+        createTree(gTmpDir, { 'src': { 'index.js': 'some code' }, 'test': { 'test.js': 'This is a README' }, 'walrus': 'animal' });
+
+        syncer.sync(gTmpDir, collectTasks, 10, function (error) {
+            expect(error).to.not.be.ok();
+            expect(gTasks.length).to.be(3);
+
+            execSync(`rm -rf src test`, { cwd: gTmpDir });
+
+            gTasks = [ ];
+            syncer.sync(gTmpDir, collectTasks, 10, function (error) {
+                expect(error).to.not.be.ok();
+
+                expect(gTasks).to.eql([
+                    { operation: 'removedir', path: 'src', reason: 'missing' },
+                    { operation: 'removedir', path: 'test', reason: 'missing' }
+                ]);
+
+                done();
+            });
+        });
+    });
+
+    it('all files disappeared', function (done) {
+        gTasks = [ ];
+        fs.writeFileSync(gCacheFile, '', 'utf8');
+        createTree(gTmpDir, { 'src': { 'index.js': 'some code' }, 'test': { 'test.js': 'This is a README' }, 'walrus': 'animal' });
+
+        syncer.sync(gTmpDir, collectTasks, 10, function (error) {
+            expect(error).to.not.be.ok();
+            expect(gTasks.length).to.be(3);
+
+            execSync(`find . -delete`, { cwd: gTmpDir });
+
+            gTasks = [ ];
+            syncer.sync(gTmpDir, collectTasks, 10, function (error) {
+                expect(error).to.not.be.ok();
+
+                expect(gTasks).to.eql([
+                    { operation: 'removedir', path: 'src', reason: 'missing' },
+                    { operation: 'removedir', path: 'test', reason: 'missing' },
+                    { operation: 'remove', path: 'walrus', reason: 'missing' }
+                ]);
+
+                done();
+            });
+        });
+    });
+
+    it('no redundant deletes', function (done) {
+        gTasks = [ ];
+        fs.writeFileSync(gCacheFile, '', 'utf8');
+        createTree(gTmpDir, { a: { b: { c: { d: { e: 'some code' } } } } });
+
+        syncer.sync(gTmpDir, collectTasks, 10, function (error) {
+            expect(error).to.not.be.ok();
+            expect(gTasks.length).to.be(1);
+
+            execSync(`rm -r a/b; touch a/f`, { cwd: gTmpDir });
+
+            gTasks = [ ];
+            syncer.sync(gTmpDir, collectTasks, 10, function (error) {
+                expect(error).to.not.be.ok();
+
+                expect(gTasks).to.eql([
+                    { operation: 'removedir', path: 'a/b', reason: 'missing' },
+                    { operation: 'add', path: 'a/f', reason: 'new' }
+                ]);
+
+                done();
+            });
+        });
+    });
+
+    it('file became dir', function (done) {
+        gTasks = [ ];
+        fs.writeFileSync(gCacheFile, '', 'utf8');
+        createTree(gTmpDir, { 'data': { 'src': { 'index.js': 'some code' }, 'test': { 'test.js': 'This is a README' }, 'walrus': 'animal' } });
+
+        syncer.sync(gTmpDir, collectTasks, 10, function (error) {
+            expect(error).to.not.be.ok();
+            expect(gTasks.length).to.be(3);
+
+            execSync(`rm data/test/test.js; mkdir data/test/test.js; touch data/test/test.js/trick`, { cwd: gTmpDir });
+
+            gTasks = [ ];
+            syncer.sync(gTmpDir, collectTasks, 10, function (error) {
+                expect(error).to.not.be.ok();
+
+                expect(gTasks).to.eql([
+                    { operation: 'remove', path: 'data/test/test.js', reason: 'wasfile' },
+                    { operation: 'add', path: 'data/test/test.js/trick', reason: 'new' }
+                ]);
+
+                done();
+            });
+        });
+    });
+
+    it('dir became file', function (done) {
+        gTasks = [ ];
+        fs.writeFileSync(gCacheFile, '', 'utf8');
+        createTree(gTmpDir, { 'src': { 'index.js': 'some code' }, 'test': { 'test.js': 'this', 'test2.js': 'test' }, 'walrus': 'animal' });
+
+        syncer.sync(gTmpDir, collectTasks, 10, function (error) {
+            expect(error).to.not.be.ok();
+            expect(gTasks.length).to.be(4);
+
+            execSync(`rm -r test; touch test`, { cwd: gTmpDir });
+
+            gTasks = [ ];
+            syncer.sync(gTmpDir, collectTasks, 10, function (error) {
+                expect(error).to.not.be.ok();
+
+                expect(gTasks).to.eql([
+                    { operation: 'removedir', path: 'test', reason: 'wasdir' },
+                    { operation: 'add', path: 'test', reason: 'wasdir' }
+                ]);
+
+                done();
+            });
+        });
+    });
+
+    it('is complicated', function (done) {
+        gTasks = [ ];
+        createTree(gTmpDir, {
+            a: 'data',
+            a2: 'data',
+            b: 'data',
+            file: 'data',
+            g: {
+                file: 'data'
+            },
+            j: {
+                k: { },
+                l: {
+                    file: 'data'
+                },
+                m: { }
+            }
+        });
+
+        syncer.sync(gTmpDir, collectTasks, 10, function (error) {
+            expect(error).to.not.be.ok();
+
+            execSync(`rm a; \
+                      mkdir a; \
+                      touch a/file; \
+                      rm a2; \
+                      touch b; \
+                      rm file g/file; \
+                      ln -s /tmp h; \
+                      rm -r j/l;
+                      touch j/k/file; \
+                      rmdir j/m;`, { cwd: gTmpDir });
+
+            gTasks = [ ];
+            syncer.sync(gTmpDir, collectTasks, 10, function (error) {
+                expect(error).to.not.be.ok();
+
+                expect(gTasks).to.eql([
+                    { operation: 'remove', path: 'a', reason: 'wasfile' },
+                    { operation: 'remove', path: 'a2', reason: 'missing' },
+                    { operation: 'remove', path: 'file', reason: 'missing' },
+                    { operation: 'remove', path: 'g/file', reason: 'missing' },
+                    { operation: 'removedir', path: 'j/l', reason: 'missing' },
+                    { operation: 'removedir', path: 'j/m', reason: 'missing' },
+
+                    { operation: 'add', path: 'a/file', reason: 'new' },
+                    { operation: 'add', path: 'b', reason: 'changed' },
+                    { operation: 'add', path: 'j/k/file', reason: 'new' },
+                ]);
+
+                gTasks = [ ];
+                syncer.sync(gTmpDir, collectTasks, 10, function (error) {
+                    expect(error).to.not.be.ok();
+                    expect(gTasks.length).to.be(0);
+
+                    done();
+                });
+            });
+        });
+    });
+});