jjkiers/cloudron-box
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Initial version of flat-file uploader

Browse Source
This commit is contained in:
Girish Ramakrishnan
2017-09-22 14:40:37 -07:00
parent 2f6cb3e913
commit 0a60365143
6 changed files with 216 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
src/storage/caas.js
View File

@@ -58,8 +58,6 @@ function upload(apiConfig, backupFilePath, sourceStream, callback) {
assert.strictEqual(typeof sourceStream, 'object');
assert.strictEqual(typeof callback, 'function');
debug('upload: %s', backupFilePath);
getBackupCredentials(apiConfig, function (error, credentials) {
if (error) return callback(error);
var params = {
@@ -174,21 +172,48 @@ function copy(apiConfig, oldFilePath, newFilePath, callback) {
getBackupCredentials(apiConfig, function (error, credentials) {
if (error) return callback(error);
var params = {
var s3 = new AWS.S3(credentials);
var listParams = {
Bucket: apiConfig.bucket,
Key: newFilePath,
CopySource: path.join(apiConfig.bucket, oldFilePath)
Prefix: oldFilePath
};
var s3 = new AWS.S3(credentials);
s3.copyObject(params, function (error) {
if (error && error.code === 'NoSuchKey') return callback(new BackupsError(BackupsError.NOT_FOUND));
if (error) {
debug('copy: s3 copy error.', error);
return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error));
}
async.forever(function listAndDelete(foreverCallback) {
s3.listObjectsV2(listParams, function (error, listData) {
if (error) {
debug('remove: Failed to list %s. Not fatal.', error);
return foreverCallback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message));
}
callback(null);
async.eachLimit(listData.Contents, 10, function copyFile(content, iteratorCallback) {
var relativePath = path.relative(oldFilePath, content.Key);
var copyParams = {
Bucket: apiConfig.bucket,
Key: path.join(newFilePath, relativePath),
CopySource: path.join(apiConfig.bucket, content.Key)
};
s3.copyObject(copyParams, function (error) {
if (error && error.code === 'NoSuchKey') return iteratorCallback(new BackupsError(BackupsError.NOT_FOUND, 'Old backup not found'));
if (error) {
debug('copy: s3 copy error.', error);
return iteratorCallback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message));
}
iteratorCallback();
});
}, function doneCopying(error) {
if (error) return foreverCallback(error);
if (listData.IsTruncated) return foreverCallback();
foreverCallback(new Error('Done'));
});
});
}, function (error) {
if (error.message === 'Done') return callback();
callback(error);
});
});
}

8
src/storage/filesystem.js
View File

@@ -34,11 +34,11 @@ function upload(apiConfig, backupFilePath, sourceStream, callback) {
assert.strictEqual(typeof sourceStream, 'object');
assert.strictEqual(typeof callback, 'function');
debug('upload: %s', backupFilePath);
mkdirp(path.dirname(backupFilePath), function (error) {
if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message));
safe.fs.unlinkSync(backupFilePath); // remove any hardlink
var fileStream = fs.createWriteStream(backupFilePath);
fileStream.on('error', function (error) {
@@ -47,11 +47,9 @@ function upload(apiConfig, backupFilePath, sourceStream, callback) {
});
fileStream.on('close', function () {
debug('[%s] upload: changing ownership.', backupFilePath);
if (!safe.child_process.execSync('chown -R ' + BACKUP_USER + ':' + BACKUP_USER + ' ' + path.dirname(backupFilePath))) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, safe.error.message));
debug('[%s] upload: done.', backupFilePath);
debug('upload %s: done.', backupFilePath);
callback(null);
});

51
src/storage/s3.js
View File

@@ -66,8 +66,6 @@ function upload(apiConfig, backupFilePath, sourceStream, callback) {
assert.strictEqual(typeof sourceStream, 'object');
assert.strictEqual(typeof callback, 'function');
debug('upload: %s', backupFilePath);
getBackupCredentials(apiConfig, function (error, credentials) {
if (error) return callback(error);
@@ -183,21 +181,48 @@ function copy(apiConfig, oldFilePath, newFilePath, callback) {
getBackupCredentials(apiConfig, function (error, credentials) {
if (error) return callback(error);
var params = {
var s3 = new AWS.S3(credentials);
var listParams = {
Bucket: apiConfig.bucket,
Key: newFilePath,
CopySource: path.join(apiConfig.bucket, oldFilePath)
Prefix: oldFilePath
};
var s3 = new AWS.S3(credentials);
s3.copyObject(params, function (error) {
if (error && error.code === 'NoSuchKey') return callback(new BackupsError(BackupsError.NOT_FOUND, 'Old backup not found'));
if (error) {
debug('copy: s3 copy error.', error);
return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message));
}
async.forever(function listAndDelete(foreverCallback) {
s3.listObjectsV2(listParams, function (error, listData) {
if (error) {
debug('remove: Failed to list %s. Not fatal.', error);
return foreverCallback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message));
}
callback(null);
async.eachLimit(listData.Contents, 10, function copyFile(content, iteratorCallback) {
var relativePath = path.relative(oldFilePath, content.Key);
var copyParams = {
Bucket: apiConfig.bucket,
Key: path.join(newFilePath, relativePath),
CopySource: path.join(apiConfig.bucket, content.Key)
};
s3.copyObject(copyParams, function (error) {
if (error && error.code === 'NoSuchKey') return iteratorCallback(new BackupsError(BackupsError.NOT_FOUND, 'Old backup not found'));
if (error) {
debug('copy: s3 copy error.', error);
return iteratorCallback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message));
}
iteratorCallback();
});
}, function doneCopying(error) {
if (error) return foreverCallback(error);
if (listData.IsTruncated) return foreverCallback();
foreverCallback(new Error('Done'));
});
});
}, function (error) {
if (error.message === 'Done') return callback();
callback(error);
});
});
}