diff --git a/src/backups.js b/src/backups.js index cebb9fbad..e93bc4c26 100644 --- a/src/backups.js +++ b/src/backups.js @@ -238,6 +238,8 @@ function sync(backupConfig, backupId, dataDir, callback) { } else if (task.operation === 'removedir') { safe.fs.writeFileSync(paths.BACKUP_RESULT_FILE, 'Removing directory ' + task.path); api(backupConfig.provider).removeDir(backupConfig, backupFilePath, iteratorCallback); + } else if (task.operation === 'mkdir') { // unused + safe.fs.writeFileSync(paths.BACKUP_RESULT_FILE, 'Adding directory ' + task.path); } }, 10 /* concurrency */, function (error) { if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message)); diff --git a/src/storage/filesystem.js b/src/storage/filesystem.js index 6123ac35b..94f037bc7 100644 --- a/src/storage/filesystem.js +++ b/src/storage/filesystem.js @@ -20,7 +20,6 @@ var assert = require('assert'), debug = require('debug')('box:storage/filesystem'), fs = require('fs'), mkdirp = require('mkdirp'), - PassThrough = require('stream').PassThrough, path = require('path'), safe = require('safetydance'), shell = require('../shell.js'); diff --git a/src/syncer.js b/src/syncer.js index 9c34d5c84..ee8a0826c 100644 --- a/src/syncer.js +++ b/src/syncer.js @@ -3,6 +3,7 @@ var assert = require('assert'), async = require('async'), debug = require('debug')('box:syncer'), + fs = require('fs'), path = require('path'), paths = require('./paths.js'), safe = require('safetydance'); @@ -29,6 +30,14 @@ function readTree(dir) { return list.map(function (e) { return { stat: safe.fs.lstatSync(path.join(dir, e)), name: e }; }); } +function ISDIR(x) { + return (x & fs.constants.S_IFDIR) === fs.constants.S_IFDIR; +} + +function ISFILE(x) { + return (x & fs.constants.S_IFREG) === fs.constants.S_IFREG; +} + function sync(dir, taskProcessor, concurrency, callback) { assert.strictEqual(typeof dir, 'string'); assert.strictEqual(typeof taskProcessor, 'function'); @@ -41,7 +50,7 @@ function sync(dir, taskProcessor, concurrency, callback) { newCacheFile = path.join(paths.BACKUP_INFO_DIR, path.basename(dir) + '.sync.cache.new'); if (!safe.fs.existsSync(cacheFile)) { // if cache is missing, start out empty. TODO: do a remote listDir and rebuild - delQueue.push({ operation: 'removedir', path: '' }); + delQueue.push({ operation: 'removedir', path: '', reason: 'nocache' }); } var cache = readCache(cacheFile); @@ -50,39 +59,65 @@ function sync(dir, taskProcessor, concurrency, callback) { if (newCacheFd === -1) return callback(new Error('Error opening new cache file: ' + safe.error.message)); function advanceCache(entryPath) { - // TODO: detect and issue removedir + var lastRemovedDir = null; + for (; curCacheIndex !== cache.length && (entryPath === '' || cache[curCacheIndex].path < entryPath); ++curCacheIndex) { - delQueue.push({ operation: 'remove', path: cache[curCacheIndex].path }); + // ignore subdirs of lastRemovedDir since it was removed already + if (lastRemovedDir && cache[curCacheIndex].path.startsWith(lastRemovedDir)) continue; + + if (ISDIR(cache[curCacheIndex].stat.mode)) { + delQueue.push({ operation: 'removedir', path: cache[curCacheIndex].path, reason: 'missing' }); + lastRemovedDir = cache[curCacheIndex].path; + } else { + delQueue.push({ operation: 'remove', path: cache[curCacheIndex].path, reason: 'missing' }); + lastRemovedDir = null; + } } } function traverse(relpath) { var entries = readTree(path.join(dir, relpath)); + // addQueue.push({ operation: 'mkdir', path: relpath }); for (var i = 0; i < entries.length; i++) { var entryPath = path.join(relpath, entries[i].name); - var stat = entries[i].stat; + var entryStat = entries[i].stat; - if (!stat) continue; // some stat error - if (!stat.isDirectory() && !stat.isFile()) continue; - if (stat.isSymbolicLink()) continue; + if (!entryStat) continue; // some stat error. prented it doesn't exist + if (!entryStat.isDirectory() && !entryStat.isFile()) continue; // ignore non-files and dirs + if (entryStat.isSymbolicLink()) continue; - if (stat.isDirectory()) { - traverse(entryPath); - continue; + safe.fs.appendFileSync(newCacheFd, JSON.stringify({ path: entryPath, stat: { mtime: entryStat.mtime.getTime(), size: entryStat.size, inode: entryStat.inode, mode: entryStat.mode } }) + '\n'); + + if (curCacheIndex !== cache.length && cache[curCacheIndex].path < entryPath) { // files disappeared. first advance cache as needed + advanceCache(entryPath); } - safe.fs.appendFileSync(newCacheFd, JSON.stringify({ path: entryPath, mtime: stat.mtime.getTime(), size: stat.size, inode: stat.inode }) + '\n'); + const cachePath = curCacheIndex === cache.length ? null : cache[curCacheIndex].path; + const cacheStat = curCacheIndex === cache.length ? null : cache[curCacheIndex].stat; - advanceCache(entryPath); - - if (curCacheIndex !== cache.length && cache[curCacheIndex].path === entryPath) { - if (stat.mtime.getTime() !== cache[curCacheIndex].mtime || stat.size != cache[curCacheIndex].size || stat.inode !== cache[curCacheIndex].inode) { - addQueue.push({ operation: 'add', path: entryPath }); + if (cachePath === null || cachePath > entryPath) { // new files appeared + if (entryStat.isDirectory()) { + traverse(entryPath); + } else { + addQueue.push({ operation: 'add', path: entryPath, reason: 'new' }); + } + } else if (ISDIR(cacheStat.mode) && entryStat.isDirectory()) { // dir names match + ++curCacheIndex; + traverse(entryPath); + } else if (ISFILE(cacheStat.mode) && entryStat.isFile()) { // file names match + if (entryStat.mtime.getTime() !== cacheStat.mtime || entryStat.size != cacheStat.size || entryStat.inode !== cacheStat.inode) { // file changed + addQueue.push({ operation: 'add', path: entryPath, reason: 'changed' }); } ++curCacheIndex; - } else { - addQueue.push({ operation: 'add', path: entryPath }); + } else if (entryStat.isDirectory()) { // was a file, now a directory + delQueue.push({ operation: 'remove', path: cachePath, reason: 'wasfile' }); + ++curCacheIndex; + traverse(entryPath); + } else { // was a dir, now a file + delQueue.push({ operation: 'removedir', path: cachePath, reason: 'wasdir' }); + while (curCacheIndex !== cache.length && cache[curCacheIndex].path.startsWith(cachePath)) ++curCacheIndex; + addQueue.push({ operation: 'add', path: entryPath, reason: 'wasdir' }); } } } diff --git a/src/test/common.js b/src/test/common.js new file mode 100644 index 000000000..5ba5b60df --- /dev/null +++ b/src/test/common.js @@ -0,0 +1,33 @@ +'use strict'; + +var fs = require('fs'), + mkdirp = require('mkdirp'), + path = require('path'), + rimraf = require('rimraf'); + +exports = module.exports = { + createTree: createTree +}; + +function createTree(root, obj) { + rimraf.sync(root); + mkdirp.sync(root); + + function createSubTree(tree, curpath) { + for (var key in tree) { + if (typeof tree[key] === 'string') { + if (key.startsWith('link:')) { + fs.symlinkSync(tree[key], path.join(curpath, key.slice(5))); + } else { + fs.writeFileSync(path.join(curpath, key), tree[key], 'utf8'); + } + } else { + fs.mkdirSync(path.join(curpath, key)); + createSubTree(tree[key], path.join(curpath, key)); + } + } + } + + createSubTree(obj, root); +} + diff --git a/src/test/syncer-test.js b/src/test/syncer-test.js new file mode 100644 index 000000000..595441cc8 --- /dev/null +++ b/src/test/syncer-test.js @@ -0,0 +1,330 @@ +/* global it:false */ +/* global describe:false */ +/* global before:false */ +/* global after:false */ + +'use strict'; + +var createTree = require('./common.js').createTree, + execSync = require('child_process').execSync, + expect = require('expect.js'), + fs = require('fs'), + os = require('os'), + path = require('path'), + paths = require('../paths.js'), + safe = require('safetydance'), + syncer = require('../syncer.js'); + +var gTasks = [ ], + gTmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'syncer-test')), + gCacheFile = path.join(paths.BACKUP_INFO_DIR, path.basename(gTmpDir) + '.sync.cache'); + +function collectTasks(task, callback) { + gTasks.push(task); + callback(); +} + +describe('Syncer', function () { + before(function () { + console.log('Tests are run in %s with cache file %s', gTmpDir, gCacheFile) + }); + + it('missing cache - removes remote dir', function (done) { + gTasks = [ ]; + safe.fs.unlinkSync(gCacheFile); + createTree(gTmpDir, { }); + + syncer.sync(gTmpDir, collectTasks, 10, function (error) { + expect(error).to.not.be.ok(); + + expect(gTasks).to.eql([ + { operation: 'removedir', path: '', reason: 'nocache' } + ]); + done(); + }); + }); + + it('empty cache - adds all', function (done) { + gTasks = [ ]; + fs.writeFileSync(gCacheFile, '', 'utf8'); + createTree(gTmpDir, { 'src': { 'index.js': 'some code' }, 'test': { 'test.js': 'This is a README' }, 'walrus': 'animal' }); + + syncer.sync(gTmpDir, collectTasks, 10, function (error) { + expect(error).to.not.be.ok(); + + expect(gTasks).to.eql([ + { operation: 'add', path: 'src/index.js', reason: 'new' }, + { operation: 'add', path: 'test/test.js', reason: 'new' }, + { operation: 'add', path: 'walrus', reason: 'new' } + ]); + done(); + }); + }); + + it('empty cache - deep', function (done) { + gTasks = [ ]; + fs.writeFileSync(gCacheFile, '', 'utf8'); + createTree(gTmpDir, { a: { b: { c: { d: { e: 'some code' } } } } }); + + syncer.sync(gTmpDir, collectTasks, 10, function (error) { + expect(error).to.not.be.ok(); + + expect(gTasks).to.eql([ + { operation: 'add', path: 'a/b/c/d/e', reason: 'new' } + ]); + done(); + }); + }); + + it('ignores special files', function (done) { + gTasks = [ ]; + fs.writeFileSync(gCacheFile, '', 'utf8'); + createTree(gTmpDir, { 'link:file': '/tmp', 'readme': 'this is readme' }); + + syncer.sync(gTmpDir, collectTasks, 10, function (error) { + expect(error).to.not.be.ok(); + + expect(gTasks).to.eql([ + { operation: 'add', path: 'readme', reason: 'new' } + ]); + done(); + }); + }); + + it('adds changed files', function (done) { + gTasks = [ ]; + fs.writeFileSync(gCacheFile, '', 'utf8'); + createTree(gTmpDir, { 'src': { 'index.js': 'some code' }, 'test': { 'test.js': 'This is a README' }, 'walrus': 'animal' }); + + syncer.sync(gTmpDir, collectTasks, 10, function (error) { + expect(error).to.not.be.ok(); + expect(gTasks.length).to.be(3); + + execSync(`touch src/index.js test/test.js`, { cwd: gTmpDir }); + + gTasks = [ ]; + syncer.sync(gTmpDir, collectTasks, 10, function (error) { + expect(error).to.not.be.ok(); + + expect(gTasks).to.eql([ + { operation: 'add', path: 'src/index.js', reason: 'changed' }, + { operation: 'add', path: 'test/test.js', reason: 'changed' } + ]); + + done(); + }); + }); + }); + + it('removes missing files', function (done) { + gTasks = [ ]; + fs.writeFileSync(gCacheFile, '', 'utf8'); + createTree(gTmpDir, { 'src': { 'index.js': 'some code' }, 'test': { 'test.js': 'This is a README' }, 'walrus': 'animal' }); + + syncer.sync(gTmpDir, collectTasks, 10, function (error) { + expect(error).to.not.be.ok(); + expect(gTasks.length).to.be(3); + + execSync(`rm src/index.js walrus`, { cwd: gTmpDir }); + + gTasks = [ ]; + syncer.sync(gTmpDir, collectTasks, 10, function (error) { + expect(error).to.not.be.ok(); + + expect(gTasks).to.eql([ + { operation: 'remove', path: 'src/index.js', reason: 'missing' }, + { operation: 'remove', path: 'walrus', reason: 'missing' } + ]); + + done(); + }); + }); + }); + + it('removes missing dirs', function (done) { + gTasks = [ ]; + fs.writeFileSync(gCacheFile, '', 'utf8'); + createTree(gTmpDir, { 'src': { 'index.js': 'some code' }, 'test': { 'test.js': 'This is a README' }, 'walrus': 'animal' }); + + syncer.sync(gTmpDir, collectTasks, 10, function (error) { + expect(error).to.not.be.ok(); + expect(gTasks.length).to.be(3); + + execSync(`rm -rf src test`, { cwd: gTmpDir }); + + gTasks = [ ]; + syncer.sync(gTmpDir, collectTasks, 10, function (error) { + expect(error).to.not.be.ok(); + + expect(gTasks).to.eql([ + { operation: 'removedir', path: 'src', reason: 'missing' }, + { operation: 'removedir', path: 'test', reason: 'missing' } + ]); + + done(); + }); + }); + }); + + it('all files disappeared', function (done) { + gTasks = [ ]; + fs.writeFileSync(gCacheFile, '', 'utf8'); + createTree(gTmpDir, { 'src': { 'index.js': 'some code' }, 'test': { 'test.js': 'This is a README' }, 'walrus': 'animal' }); + + syncer.sync(gTmpDir, collectTasks, 10, function (error) { + expect(error).to.not.be.ok(); + expect(gTasks.length).to.be(3); + + execSync(`find . -delete`, { cwd: gTmpDir }); + + gTasks = [ ]; + syncer.sync(gTmpDir, collectTasks, 10, function (error) { + expect(error).to.not.be.ok(); + + expect(gTasks).to.eql([ + { operation: 'removedir', path: 'src', reason: 'missing' }, + { operation: 'removedir', path: 'test', reason: 'missing' }, + { operation: 'remove', path: 'walrus', reason: 'missing' } + ]); + + done(); + }); + }); + }); + + it('no redundant deletes', function (done) { + gTasks = [ ]; + fs.writeFileSync(gCacheFile, '', 'utf8'); + createTree(gTmpDir, { a: { b: { c: { d: { e: 'some code' } } } } }); + + syncer.sync(gTmpDir, collectTasks, 10, function (error) { + expect(error).to.not.be.ok(); + expect(gTasks.length).to.be(1); + + execSync(`rm -r a/b; touch a/f`, { cwd: gTmpDir }); + + gTasks = [ ]; + syncer.sync(gTmpDir, collectTasks, 10, function (error) { + expect(error).to.not.be.ok(); + + expect(gTasks).to.eql([ + { operation: 'removedir', path: 'a/b', reason: 'missing' }, + { operation: 'add', path: 'a/f', reason: 'new' } + ]); + + done(); + }); + }); + }); + + it('file became dir', function (done) { + gTasks = [ ]; + fs.writeFileSync(gCacheFile, '', 'utf8'); + createTree(gTmpDir, { 'data': { 'src': { 'index.js': 'some code' }, 'test': { 'test.js': 'This is a README' }, 'walrus': 'animal' } }); + + syncer.sync(gTmpDir, collectTasks, 10, function (error) { + expect(error).to.not.be.ok(); + expect(gTasks.length).to.be(3); + + execSync(`rm data/test/test.js; mkdir data/test/test.js; touch data/test/test.js/trick`, { cwd: gTmpDir }); + + gTasks = [ ]; + syncer.sync(gTmpDir, collectTasks, 10, function (error) { + expect(error).to.not.be.ok(); + + expect(gTasks).to.eql([ + { operation: 'remove', path: 'data/test/test.js', reason: 'wasfile' }, + { operation: 'add', path: 'data/test/test.js/trick', reason: 'new' } + ]); + + done(); + }); + }); + }); + + it('dir became file', function (done) { + gTasks = [ ]; + fs.writeFileSync(gCacheFile, '', 'utf8'); + createTree(gTmpDir, { 'src': { 'index.js': 'some code' }, 'test': { 'test.js': 'this', 'test2.js': 'test' }, 'walrus': 'animal' }); + + syncer.sync(gTmpDir, collectTasks, 10, function (error) { + expect(error).to.not.be.ok(); + expect(gTasks.length).to.be(4); + + execSync(`rm -r test; touch test`, { cwd: gTmpDir }); + + gTasks = [ ]; + syncer.sync(gTmpDir, collectTasks, 10, function (error) { + expect(error).to.not.be.ok(); + + expect(gTasks).to.eql([ + { operation: 'removedir', path: 'test', reason: 'wasdir' }, + { operation: 'add', path: 'test', reason: 'wasdir' } + ]); + + done(); + }); + }); + }); + + it('is complicated', function (done) { + gTasks = [ ]; + createTree(gTmpDir, { + a: 'data', + a2: 'data', + b: 'data', + file: 'data', + g: { + file: 'data' + }, + j: { + k: { }, + l: { + file: 'data' + }, + m: { } + } + }); + + syncer.sync(gTmpDir, collectTasks, 10, function (error) { + expect(error).to.not.be.ok(); + + execSync(`rm a; \ + mkdir a; \ + touch a/file; \ + rm a2; \ + touch b; \ + rm file g/file; \ + ln -s /tmp h; \ + rm -r j/l; + touch j/k/file; \ + rmdir j/m;`, { cwd: gTmpDir }); + + gTasks = [ ]; + syncer.sync(gTmpDir, collectTasks, 10, function (error) { + expect(error).to.not.be.ok(); + + expect(gTasks).to.eql([ + { operation: 'remove', path: 'a', reason: 'wasfile' }, + { operation: 'remove', path: 'a2', reason: 'missing' }, + { operation: 'remove', path: 'file', reason: 'missing' }, + { operation: 'remove', path: 'g/file', reason: 'missing' }, + { operation: 'removedir', path: 'j/l', reason: 'missing' }, + { operation: 'removedir', path: 'j/m', reason: 'missing' }, + + { operation: 'add', path: 'a/file', reason: 'new' }, + { operation: 'add', path: 'b', reason: 'changed' }, + { operation: 'add', path: 'j/k/file', reason: 'new' }, + ]); + + gTasks = [ ]; + syncer.sync(gTmpDir, collectTasks, 10, function (error) { + expect(error).to.not.be.ok(); + expect(gTasks.length).to.be(0); + + done(); + }); + }); + }); + }); +});