jjkiers/cloudron-box
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Revert "use node-tar for extract"

Browse Source 
This reverts commit 285feb4f8b.
This commit is contained in:
Girish Ramakrishnan
2024-07-05 09:26:38 +02:00
parent 285feb4f8b
commit b64b513b14
3 changed files with 56 additions and 385 deletions
Download Patch File Download Diff File Expand all files Collapse all files

76
src/backupformat/tgz.js
View File

@@ -16,10 +16,8 @@ const assert = require('assert'),
path = require('path'),
ProgressStream = require('../progress-stream.js'),
promiseRetry = require('../promise-retry.js'),
stream = require('stream/promises'),
storage = require('../storage.js'),
tar = require('tar-fs'),
tar2 = require('tar'),
zlib = require('zlib');
function getBackupFilePath(backupConfig, remotePath) {
@@ -84,43 +82,56 @@ function tarPack(dataLayout, encryption) {
return ps;
}
async function tarExtract(inStream, dataLayout, encryption, progressCallback) {
function tarExtract(inStream, dataLayout, encryption) {
assert.strictEqual(typeof inStream, 'object');
assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
assert.strictEqual(typeof encryption, 'object');
assert.strictEqual(typeof progressCallback, 'function');
const gunzip = zlib.createGunzip({});
const ps = new ProgressStream({ interval: 10000 }); // display a progress every 10 seconds
const extract = tar2.extract({
cwd: '/tmp',
dmode: 500, // ensure directory is writable
preserveOwner: false, // use uid/gid of current process
strict: true, // error on warnings,
preservePaths: true, // allow absolute paths. otherwise will sandbox to cwd
// to map paths, we cannot use transform hook because it is only used for files and not directories
// we can use filter hook as well - https://github.com/isaacs/node-tar/issues/357#issuecomment-1416491212
onReadEntry(entry) {
// debug(entry.header.path, entry.header.type, entry.header.size);
entry.path = dataLayout.toLocalPath(entry.header.path);
},
onwarn(code, message /*, data */) {
debug(`extract warning:${message} ${code}`);
const extract = tar.extract('/', {
map: function (header) {
header.name = dataLayout.toLocalPath(header.name);
return header;
},
dmode: 500 // ensure directory is writable
});
ps.on('progress', function (progress) {
const transferred = Math.round(progress.transferred/1024/1024), speed = Math.round(progress.speed/1024/1024);
if (!transferred && !speed) return progressCallback({ message: 'Downloading backup' }); // 0M@0MBps looks wrong
progressCallback({ message: `Downloading ${transferred}M@${speed}MBps` });
const emitError = once((error) => {
inStream.destroy();
ps.emit('error', error);
});
inStream.on('error', function (error) {
debug('tarExtract: input stream error. %o', error);
emitError(new BoxError(BoxError.EXTERNAL_ERROR, error.message));
});
gunzip.on('error', function (error) {
debug('tarExtract: gunzip stream error. %o', error);
emitError(new BoxError(BoxError.EXTERNAL_ERROR, error.message));
});
extract.on('error', function (error) {
debug('tarExtract: extract stream error. %o', error);
emitError(new BoxError(BoxError.EXTERNAL_ERROR, error.message));
});
extract.on('finish', function () {
debug('tarExtract: done.');
// we use a separate event because ps is a through2 stream which emits 'finish' event indicating end of inStream and not extract
ps.emit('done');
});
if (encryption) {
const decrypt = new DecryptStream(encryption);
await stream.pipeline(inStream, ps, decrypt, extract);
decrypt.on('error', function (error) {
debug('tarExtract: decrypt stream error.', error);
emitError(new BoxError(BoxError.EXTERNAL_ERROR, `Failed to decrypt: ${error.message}`));
});
inStream.pipe(ps).pipe(decrypt).pipe(gunzip).pipe(extract);
} else {
await stream.pipeline(inStream, ps, extract);
inStream.pipe(ps).pipe(gunzip).pipe(extract);
}
return ps;
@@ -138,8 +149,19 @@ async function download(backupConfig, remotePath, dataLayout, progressCallback)
await promiseRetry({ times: 5, interval: 20000, debug }, async () => {
progressCallback({ message: `Downloading backup ${backupFilePath}` });
const sourceStream = await storage.api(backupConfig.provider).download(backupConfig, backupFilePath);
await tarExtract(sourceStream, dataLayout, backupConfig.encryption, progressCallback);
const ps = tarExtract(sourceStream, dataLayout, backupConfig.encryption);
return await new Promise((resolve, reject) => {
ps.on('progress', function (progress) {
const transferred = Math.round(progress.transferred/1024/1024), speed = Math.round(progress.speed/1024/1024);
if (!transferred && !speed) return progressCallback({ message: 'Downloading backup' }); // 0M@0MBps looks wrong
progressCallback({ message: `Downloading ${transferred}M@${speed}MBps` });
});
ps.on('error', reject);
ps.on('done', resolve);
});
});
}