backups: remove entries from database that don't exist in storage

fixes #772

src/storage/filesystem.js

@@ -1,22 +1,23 @@
 'use strict';
 
 exports = module.exports = {
-    getBackupPath: getBackupPath,
-    checkPreconditions: checkPreconditions,
+    getBackupPath,
+    checkPreconditions,
 
-    upload: upload,
-    download: download,
+    upload,
+    download,
 
-    copy: copy,
+    copy,
 
-    listDir: listDir,
+    exists,
+    listDir,
 
-    remove: remove,
-    removeDir: removeDir,
+    remove,
+    removeDir,
 
-    testConfig: testConfig,
-    removePrivateFields: removePrivateFields,
-    injectPrivateFields: injectPrivateFields
+    testConfig,
+    removePrivateFields,
+    injectPrivateFields
 };
 
 const PROVIDER_FILESYSTEM = 'filesystem';
@@ -136,6 +137,20 @@ function download(apiConfig, sourceFilePath, callback) {
     callback(null, fileStream);
 }
 
+function exists(apiConfig, sourceFilePath, callback) {
+    assert.strictEqual(typeof apiConfig, 'object');
+    assert.strictEqual(typeof sourceFilePath, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    // do not use existsSync because it does not return EPERM etc
+    if (!safe.fs.statSync(sourceFilePath)) {
+        if (safe.error && safe.error.code === 'ENOENT') return callback(null, false);
+        if (safe.error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Exists ${sourceFilePath}: ${safe.error.message}`));
+    }
+
+    callback(null, true);
+}
+
 function listDir(apiConfig, dir, batchSize, iteratorCallback, callback) {
     assert.strictEqual(typeof apiConfig, 'object');
     assert.strictEqual(typeof dir, 'string');

src/storage/gcs.js

@@ -5,6 +5,7 @@ exports = module.exports = {
     checkPreconditions,
 
     upload,
+    exists,
     download,
     copy,
 
@@ -100,6 +101,36 @@ function upload(apiConfig, backupFilePath, sourceStream, callback) {
     sourceStream.pipe(uploadStream);
 }
 
+function exists(apiConfig, backupFilePath, callback) {
+    assert.strictEqual(typeof apiConfig, 'object');
+    assert.strictEqual(typeof backupFilePath, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    const bucket = getBucket(apiConfig);
+
+    if (!backupFilePath.endsWith('/')) {
+        const file = bucket.file(backupFilePath);
+        file.getMetadata(function (error) {
+            if (error && error.code === 404) return callback(null, false);
+            if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, error.message));
+
+            callback(null, true);
+        });
+    } else {
+        const query = {
+            prefix: backupFilePath,
+            maxResults: 1,
+            autoPaginate: true
+        };
+
+        bucket.getFiles(query, function (error, files) {
+            if (error) return callback(error);
+
+            callback(null, files.length !== 0);
+        });
+    }
+}
+
 function download(apiConfig, backupFilePath, callback) {
     assert.strictEqual(typeof apiConfig, 'object');
     assert.strictEqual(typeof backupFilePath, 'string');

src/storage/interface.js

@@ -11,23 +11,25 @@
 //     for the other API calls we leave it to the backend to retry. this allows
 //     them to tune the concurrency based on failures/rate limits accordingly
 exports = module.exports = {
-    getBackupPath: getBackupPath,
-    checkPreconditions: checkPreconditions,
+    getBackupPath,
+    checkPreconditions,
 
-    upload: upload,
+    upload,
 
-    download: download,
-    downloadDir: downloadDir,
-    copy: copy,
+    exists,
 
-    listDir: listDir,
+    download,
+    downloadDir,
+    copy,
 
-    remove: remove,
-    removeDir: removeDir,
+    listDir,
 
-    testConfig: testConfig,
-    removePrivateFields: removePrivateFields,
-    injectPrivateFields: injectPrivateFields
+    remove,
+    removeDir,
+
+    testConfig,
+    removePrivateFields,
+    injectPrivateFields
 };
 
 var assert = require('assert'),
@@ -72,6 +74,14 @@ function upload(apiConfig, backupFilePath, sourceStream, callback) {
     callback(new BoxError(BoxError.NOT_IMPLEMENTED, 'upload is not implemented'));
 }
 
+function exists(apiConfig, backupFilePath, callback) {
+    assert.strictEqual(typeof apiConfig, 'object');
+    assert.strictEqual(typeof backupFilePath, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    callback(new BoxError(BoxError.NOT_IMPLEMENTED, 'exists is not implemented'));
+}
+
 function download(apiConfig, backupFilePath, callback) {
     assert.strictEqual(typeof apiConfig, 'object');
     assert.strictEqual(typeof backupFilePath, 'string');

src/storage/noop.js

@@ -1,22 +1,23 @@
 'use strict';
 
 exports = module.exports = {
-    getBackupPath: getBackupPath,
-    checkPreconditions: checkPreconditions,
+    getBackupPath,
+    checkPreconditions,
 
-    upload: upload,
-    download: download,
-    downloadDir: downloadDir,
-    copy: copy,
+    upload,
+    exists,
+    download,
+    downloadDir,
+    copy,
 
-    listDir: listDir,
+    listDir,
 
-    remove: remove,
-    removeDir: removeDir,
+    remove,
+    removeDir,
 
-    testConfig: testConfig,
-    removePrivateFields: removePrivateFields,
-    injectPrivateFields: injectPrivateFields
+    testConfig,
+    removePrivateFields,
+    injectPrivateFields
 };
 
 var assert = require('assert'),
@@ -49,6 +50,16 @@ function upload(apiConfig, backupFilePath, sourceStream, callback) {
     callback(null);
 }
 
+function exists(apiConfig, backupFilePath, callback) {
+    assert.strictEqual(typeof apiConfig, 'object');
+    assert.strictEqual(typeof backupFilePath, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('exists: %s', backupFilePath);
+
+    callback(null, false);
+}
+
 function download(apiConfig, backupFilePath, callback) {
     assert.strictEqual(typeof apiConfig, 'object');
     assert.strictEqual(typeof backupFilePath, 'string');

src/storage/s3.js

@@ -5,6 +5,7 @@ exports = module.exports = {
     checkPreconditions,
 
     upload,
+    exists,
     download,
     copy,
 
@@ -56,7 +57,7 @@ function getS3Config(apiConfig, callback) {
     assert.strictEqual(typeof apiConfig, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    var credentials = {
+    let credentials = {
         signatureVersion: apiConfig.signatureVersion || 'v4',
         s3ForcePathStyle: false, // Use vhost style instead of path style - https://forums.aws.amazon.com/ann.jspa?annID=6776
         accessKeyId: apiConfig.accessKeyId,
@@ -137,6 +138,44 @@ function upload(apiConfig, backupFilePath, sourceStream, callback) {
     });
 }
 
+function exists(apiConfig, backupFilePath, callback) {
+    assert.strictEqual(typeof apiConfig, 'object');
+    assert.strictEqual(typeof backupFilePath, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    getS3Config(apiConfig, function (error, credentials) {
+        if (error) return callback(error);
+
+        const s3 = new AWS.S3(credentials);
+
+        if (!backupFilePath.endsWith('/')) { // check for file
+            const params = {
+                Bucket: apiConfig.bucket,
+                Key: backupFilePath
+            };
+
+            s3.headObject(params, function (error) {
+                if (error && S3_NOT_FOUND(error)) return callback(null, false);
+                if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, error.message || error.code));
+
+                callback(null, true);
+            });
+        } else { // list dir contents
+            const listParams = {
+                Bucket: apiConfig.bucket,
+                Prefix: backupFilePath,
+                MaxKeys: 1
+            };
+
+            s3.listObjects(listParams, function (error, listData) {
+                if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, error.message || error.code));
+
+                callback(null, listData.Contents.length !== 0);
+            });
+        }
+    });
+}
+
 function download(apiConfig, backupFilePath, callback) {
     assert.strictEqual(typeof apiConfig, 'object');
     assert.strictEqual(typeof backupFilePath, 'string');