From 9e5b3b4b92dfa20b625a4fb3fedf16242b252536 Mon Sep 17 00:00:00 2001 From: Ziggy Jonsson Date: Sat, 11 May 2024 15:48:39 -0400 Subject: [PATCH 1/7] eslint --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index eefdaf4..3f3f108 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -35,4 +35,5 @@ jobs: node-version: ${{ matrix.node-version }} - run: npm install - run: npm run build --if-present + - run: npx eslint . - run: npm test From ff7f8a0a9cc664483ca6f00c2aa210b708cac398 Mon Sep 17 00:00:00 2001 From: Ziggy Jonsson Date: Sat, 11 May 2024 15:54:52 -0400 Subject: [PATCH 2/7] only run on 18 --- .github/workflows/test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3f3f108..eefdaf4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -35,5 +35,4 @@ jobs: node-version: ${{ matrix.node-version }} - run: npm install - run: npm run build --if-present - - run: npx eslint . - run: npm test From 2be696cbcdaec7e458022fb1000874ce3dd417b6 Mon Sep 17 00:00:00 2001 From: Ziggy Jonsson Date: Sat, 11 May 2024 13:05:07 -0400 Subject: [PATCH 3/7] Move open to async/await --- lib/Open/directory.js | 337 ++++++++++++++++++++---------------------- lib/Open/unzip.js | 87 +++++------ 2 files changed, 202 insertions(+), 222 deletions(-) diff --git a/lib/Open/directory.js b/lib/Open/directory.js index 88ea27d..98616b8 100644 --- a/lib/Open/directory.js +++ b/lib/Open/directory.js @@ -11,29 +11,27 @@ const Bluebird = require('bluebird'); const signature = Buffer.alloc(4); signature.writeUInt32LE(0x06054b50, 0); -function getCrxHeader(source) { - const sourceStream = source.stream(0).pipe(PullStream()); - - return sourceStream.pull(4).then(function(data) { - const signature = data.readUInt32LE(0); - if (signature === 0x34327243) { - let crxHeader; - return sourceStream.pull(12).then(function(data) { - crxHeader = parseBuffer.parse(data, [ - ['version', 4], - ['pubKeyLength', 4], - ['signatureLength', 4], - ]); - }).then(function() { - return sourceStream.pull(crxHeader.pubKeyLength +crxHeader.signatureLength); - }).then(function(data) { - crxHeader.publicKey = data.slice(0, crxHeader.pubKeyLength); - crxHeader.signature = data.slice(crxHeader.pubKeyLength); - crxHeader.size = 16 + crxHeader.pubKeyLength +crxHeader.signatureLength; - return crxHeader; - }); - } - }); +async function getCrxHeader(source) { + var sourceStream = source.stream(0).pipe(PullStream()); + + let data = await sourceStream.pull(4); + var signature = data.readUInt32LE(0); + if (signature === 0x34327243) { + var crxHeader; + data = await sourceStream.pull(12); + crxHeader = parseBuffer.parse(data, [ + ['version', 4], + ['pubKeyLength', 4], + ['signatureLength', 4], + ]); + + data = await sourceStream.pull(crxHeader.pubKeyLength +crxHeader.signatureLength); + + crxHeader.publicKey = data.slice(0,crxHeader.pubKeyLength); + crxHeader.signature = data.slice(crxHeader.pubKeyLength); + crxHeader.size = 16 + crxHeader.pubKeyLength +crxHeader.signatureLength; + return crxHeader; + } } // Zip64 File Format Notes: https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT @@ -77,161 +75,148 @@ function parseZip64DirRecord (dir64record) { return vars; } -module.exports = function centralDirectory(source, options) { - const endDir = PullStream(); - const records = PullStream(); - const tailSize = (options && options.tailSize) || 80; - let sourceSize, - crxHeader, - startOffset, - vars; +module.exports = async function centralDirectory(source, options) { + var endDir = PullStream(), + records = PullStream(), + tailSize = (options && options.tailSize) || 80, + crxHeader, + startOffset, + vars; if (options && options.crx) - crxHeader = getCrxHeader(source); - - return source.size() - .then(function(size) { - sourceSize = size; - - source.stream(Math.max(0, size-tailSize)) - .on('error', function (error) { endDir.emit('error', error); }) - .pipe(endDir); - - return endDir.pull(signature); - }) - .then(function() { - return Bluebird.props({directory: endDir.pull(22), crxHeader: crxHeader}); - }) - .then(function(d) { - const data = d.directory; - startOffset = d.crxHeader && d.crxHeader.size || 0; - - vars = parseBuffer.parse(data, [ - ['signature', 4], - ['diskNumber', 2], - ['diskStart', 2], - ['numberOfRecordsOnDisk', 2], - ['numberOfRecords', 2], - ['sizeOfCentralDirectory', 4], - ['offsetToStartOfCentralDirectory', 4], - ['commentLength', 2], - ]); - - // Is this zip file using zip64 format? Use same check as Go: - // https://github.com/golang/go/blob/master/src/archive/zip/reader.go#L503 - // For zip64 files, need to find zip64 central directory locator header to extract - // relative offset for zip64 central directory record. - if (vars.numberOfRecords == 0xffff|| vars.numberOfRecords == 0xffff || - vars.offsetToStartOfCentralDirectory == 0xffffffff) { - - // Offset to zip64 CDL is 20 bytes before normal CDR - const zip64CDLSize = 20; - const zip64CDLOffset = sourceSize - (tailSize - endDir.match + zip64CDLSize); - const zip64CDLStream = PullStream(); - - source.stream(zip64CDLOffset).pipe(zip64CDLStream); - - return zip64CDLStream.pull(zip64CDLSize) - .then(function (d) { return getZip64CentralDirectory(source, d); }) - .then(function (dir64record) { - vars = parseZip64DirRecord(dir64record); - }); - } else { - vars.offsetToStartOfCentralDirectory += startOffset; + crxHeader = await getCrxHeader(source); + + const sourceSize = await source.size(); + + source.stream(Math.max(0,sourceSize-tailSize)) + .on('error', function (error) { endDir.emit('error', error) }) + .pipe(endDir); + + await endDir.pull(signature); + + var data = await endDir.pull(22); + startOffset = crxHeader && crxHeader.size || 0; + + vars = parseBuffer.parse(data, [ + ['signature', 4], + ['diskNumber', 2], + ['diskStart', 2], + ['numberOfRecordsOnDisk', 2], + ['numberOfRecords', 2], + ['sizeOfCentralDirectory', 4], + ['offsetToStartOfCentralDirectory', 4], + ['commentLength', 2], + ]); + + // Is this zip file using zip64 format? Use same check as Go: + // https://github.com/golang/go/blob/master/src/archive/zip/reader.go#L503 + // For zip64 files, need to find zip64 central directory locator header to extract + // relative offset for zip64 central directory record. + if (vars.numberOfRecords == 0xffff|| vars.numberOfRecords == 0xffff || + vars.offsetToStartOfCentralDirectory == 0xffffffff) { + + // Offset to zip64 CDL is 20 bytes before normal CDR + const zip64CDLSize = 20 + const zip64CDLOffset = sourceSize - (tailSize - endDir.match + zip64CDLSize) + const zip64CDLStream = PullStream(); + + source.stream(zip64CDLOffset).pipe(zip64CDLStream); + + const d = await zip64CDLStream.pull(zip64CDLSize) + const dir64record = await getZip64CentralDirectory(source, d);; + + vars = parseZip64DirRecord(dir64record) + + } else { + vars.offsetToStartOfCentralDirectory += startOffset; + } + + if (vars.commentLength) { + const comment = await endDir.pull(vars.commentLength); + vars.comment = comment.toString('utf8'); + }; + + source.stream(vars.offsetToStartOfCentralDirectory).pipe(records); + + vars.extract = async function(opts) { + if (!opts || !opts.path) throw new Error('PATH_MISSING'); + // make sure path is normalized before using it + opts.path = path.resolve(path.normalize(opts.path)); + const files = await vars.files; + + return Promise.map(files, function(entry) { + if (entry.type == 'Directory') return; + + // to avoid zip slip (writing outside of the destination), we resolve + // the target path, and make sure it's nested in the intended + // destination, or not extract it otherwise. + var extractPath = path.join(opts.path, entry.path); + if (extractPath.indexOf(opts.path) != 0) { + return; } - }) - .then(function() { - if (vars.commentLength) return endDir.pull(vars.commentLength).then(function(comment) { - vars.comment = comment.toString('utf8'); - }); - }) - .then(function() { - source.stream(vars.offsetToStartOfCentralDirectory).pipe(records); - - vars.extract = function(opts) { - if (!opts || !opts.path) throw new Error('PATH_MISSING'); - // make sure path is normalized before using it - opts.path = path.resolve(path.normalize(opts.path)); - return vars.files.then(function(files) { - return Bluebird.map(files, function(entry) { - if (entry.type == 'Directory') return; - - // to avoid zip slip (writing outside of the destination), we resolve - // the target path, and make sure it's nested in the intended - // destination, or not extract it otherwise. - const extractPath = path.join(opts.path, entry.path); - if (extractPath.indexOf(opts.path) != 0) { - return; - } - const writer = opts.getWriter ? opts.getWriter({path: extractPath}) : Writer({ path: extractPath }); - - return new Promise(function(resolve, reject) { - entry.stream(opts.password) - .on('error', reject) - .pipe(writer) - .on('close', resolve) - .on('error', reject); - }); - }, { concurrency: opts.concurrency > 1 ? opts.concurrency : 1 }); - }); - }; - - vars.files = Bluebird.mapSeries(Array(vars.numberOfRecords), function() { - return records.pull(46).then(function(data) { - const vars = parseBuffer.parse(data, [ - ['signature', 4], - ['versionMadeBy', 2], - ['versionsNeededToExtract', 2], - ['flags', 2], - ['compressionMethod', 2], - ['lastModifiedTime', 2], - ['lastModifiedDate', 2], - ['crc32', 4], - ['compressedSize', 4], - ['uncompressedSize', 4], - ['fileNameLength', 2], - ['extraFieldLength', 2], - ['fileCommentLength', 2], - ['diskNumber', 2], - ['internalFileAttributes', 2], - ['externalFileAttributes', 4], - ['offsetToLocalFileHeader', 4], - ]); - - vars.offsetToLocalFileHeader += startOffset; - vars.lastModifiedDateTime = parseDateTime(vars.lastModifiedDate, vars.lastModifiedTime); - - return records.pull(vars.fileNameLength).then(function(fileNameBuffer) { - vars.pathBuffer = fileNameBuffer; - vars.path = fileNameBuffer.toString('utf8'); - vars.isUnicode = (vars.flags & 0x800) != 0; - return records.pull(vars.extraFieldLength); - }) - .then(function(extraField) { - vars.extra = parseExtraField(extraField, vars); - return records.pull(vars.fileCommentLength); - }) - .then(function(comment) { - vars.comment = comment; - vars.type = (vars.uncompressedSize === 0 && /[/\\]$/.test(vars.path)) ? 'Directory' : 'File'; - const padding = options && options.padding || 1000; - vars.stream = function(_password) { - const totalSize = 30 - + padding // add an extra buffer - + (vars.extraFieldLength || 0) - + (vars.fileNameLength || 0) - + vars.compressedSize; - - return unzip(source, vars.offsetToLocalFileHeader, _password, vars, totalSize); - }; - vars.buffer = function(_password) { - return BufferStream(vars.stream(_password)); - }; - return vars; - }); - }); + var writer = opts.getWriter ? opts.getWriter({path: extractPath}) : Writer({ path: extractPath }); + + return new Promise(function(resolve, reject) { + entry.stream(opts.password) + .on('error',reject) + .pipe(writer) + .on('close',resolve) + .on('error',reject); }); - - return Bluebird.props(vars); - }); + }, { concurrency: opts.concurrency > 1 ? opts.concurrency : 1 }); + }; + + vars.files = Promise.mapSeries(Array(vars.numberOfRecords),async function() { + const data = await records.pull(46) + var vars = vars = parseBuffer.parse(data, [ + ['signature', 4], + ['versionMadeBy', 2], + ['versionsNeededToExtract', 2], + ['flags', 2], + ['compressionMethod', 2], + ['lastModifiedTime', 2], + ['lastModifiedDate', 2], + ['crc32', 4], + ['compressedSize', 4], + ['uncompressedSize', 4], + ['fileNameLength', 2], + ['extraFieldLength', 2], + ['fileCommentLength', 2], + ['diskNumber', 2], + ['internalFileAttributes', 2], + ['externalFileAttributes', 4], + ['offsetToLocalFileHeader', 4], + ]); + + vars.offsetToLocalFileHeader += startOffset; + vars.lastModifiedDateTime = parseDateTime(vars.lastModifiedDate, vars.lastModifiedTime); + + const fileNameBuffer = await records.pull(vars.fileNameLength); + vars.pathBuffer = fileNameBuffer; + vars.path = fileNameBuffer.toString('utf8'); + vars.isUnicode = (vars.flags & 0x800) != 0; + const extraField = await records.pull(vars.extraFieldLength); + + vars.extra = parseExtraField(extraField, vars); + const comment = await records.pull(vars.fileCommentLength); + + vars.comment = comment; + vars.type = (vars.uncompressedSize === 0 && /[\/\\]$/.test(vars.path)) ? 'Directory' : 'File'; + var padding = options && options.padding || 1000; + vars.stream = function(_password) { + var totalSize = 30 + + padding // add an extra buffer + + (vars.extraFieldLength || 0) + + (vars.fileNameLength || 0) + + vars.compressedSize; + + return unzip(source, vars.offsetToLocalFileHeader,_password, vars, totalSize); + }; + vars.buffer = function(_password) { + return BufferStream(vars.stream(_password)); + }; + return vars; + }); + + return Promise.props(vars); }; diff --git a/lib/Open/unzip.js b/lib/Open/unzip.js index b8d55bf..5cb0ef1 100644 --- a/lib/Open/unzip.js +++ b/lib/Open/unzip.js @@ -15,8 +15,11 @@ module.exports = function unzip(source, offset, _password, directoryVars, length entry.emit('error', e); }); - entry.vars = file.pull(30) - .then(function(data) { + // Create a separate promise chain to pipe into entry + // This allows us to return entry synchronously + Promise.resolve() + .then(async function () { + const data = await file.pull(30); let vars = parseBuffer.parse(data, [ ['signature', 4], ['versionsNeededToExtract', 2], @@ -33,52 +36,44 @@ module.exports = function unzip(source, offset, _password, directoryVars, length vars.lastModifiedDateTime = parseDateTime(vars.lastModifiedDate, vars.lastModifiedTime); - return file.pull(vars.fileNameLength) - .then(function(fileName) { - vars.fileName = fileName.toString('utf8'); - return file.pull(vars.extraFieldLength); - }) - .then(function(extraField) { - let checkEncryption; - vars.extra = parseExtraField(extraField, vars); - // Ignore logal file header vars if the directory vars are available - if (directoryVars && directoryVars.compressedSize) vars = directoryVars; - - if (vars.flags & 0x01) checkEncryption = file.pull(12) - .then(function(header) { - if (!_password) - throw new Error('MISSING_PASSWORD'); - - const decrypt = Decrypt(); - - String(_password).split('').forEach(function(d) { - decrypt.update(d); - }); - - for (let i=0; i < header.length; i++) - header[i] = decrypt.decryptByte(header[i]); - - vars.decrypt = decrypt; - vars.compressedSize -= 12; - - const check = (vars.flags & 0x8) ? (vars.lastModifiedTime >> 8) & 0xff : (vars.crc32 >> 24) & 0xff; - if (header[11] !== check) - throw new Error('BAD_PASSWORD'); - - return vars; - }); - - return Promise.resolve(checkEncryption) - .then(function() { - entry.emit('vars', vars); - return vars; - }); + const fileName = await file.pull(vars.fileNameLength); + + vars.fileName = fileName.toString('utf8'); + const extraField = await file.pull(vars.extraFieldLength); + + var checkEncryption; + vars.extra = parseExtraField(extraField, vars); + // Ignore logal file header vars if the directory vars are available + if (directoryVars && directoryVars.compressedSize) vars = directoryVars; + + if (vars.flags & 0x01) { + const header = await file.pull(12) + + if (!_password) + throw new Error('MISSING_PASSWORD'); + + var decrypt = Decrypt(); + + String(_password).split('').forEach(function(d) { + decrypt.update(d); }); - }); - entry.vars.then(function(vars) { - const fileSizeKnown = !(vars.flags & 0x08) || vars.compressedSize > 0; - let eof; + for (var i=0; i < header.length; i++) + header[i] = decrypt.decryptByte(header[i]); + + vars.decrypt = decrypt; + vars.compressedSize -= 12; + + var check = (vars.flags & 0x8) ? (vars.lastModifiedTime >> 8) & 0xff : (vars.crc32 >> 24) & 0xff; + if (header[11] !== check) + throw new Error('BAD_PASSWORD'); + }; + + + entry.emit('vars',vars); + + var fileSizeKnown = !(vars.flags & 0x08) || vars.compressedSize > 0, + eof; const inflater = vars.compressionMethod ? zlib.createInflateRaw() : Stream.PassThrough(); From c80fd62cd8d387da247ea6b1802c34ae14ee838e Mon Sep 17 00:00:00 2001 From: Ziggy Jonsson Date: Sat, 11 May 2024 14:38:51 -0400 Subject: [PATCH 4/7] Retry stream if not big enough --- lib/Open/unzip.js | 17 ++++++++++++++--- test/office-files.js | 26 ++++++++++++++++++++++++-- 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/lib/Open/unzip.js b/lib/Open/unzip.js index 5cb0ef1..079ca7d 100644 --- a/lib/Open/unzip.js +++ b/lib/Open/unzip.js @@ -6,9 +6,9 @@ const parseExtraField = require('../parseExtraField'); const parseDateTime = require('../parseDateTime'); const parseBuffer = require('../parseBuffer'); -module.exports = function unzip(source, offset, _password, directoryVars, length) { - const file = PullStream(), - entry = Stream.PassThrough(); +module.exports = function unzip(source, offset, _password, directoryVars, length, _entry) { + var file = PullStream(), + entry = _entry || Stream.PassThrough(); const req = source.stream(offset, length); req.pipe(file).on('error', function(e) { @@ -34,6 +34,17 @@ module.exports = function unzip(source, offset, _password, directoryVars, length ['extraFieldLength', 2], ]); + var localSize = 30 + + 100 // add extra padding + + (vars.extraFieldLength || 0) + + (vars.fileNameLength || 0) + + vars.compressedSize; + + if (localSize > length) { + entry.emit('streamRetry', localSize); + return unzip(source, offset, _password, directoryVars, localSize, entry); + } + vars.lastModifiedDateTime = parseDateTime(vars.lastModifiedDate, vars.lastModifiedTime); const fileName = await file.pull(vars.fileNameLength); diff --git a/test/office-files.js b/test/office-files.js index b7baabe..00068e2 100644 --- a/test/office-files.js +++ b/test/office-files.js @@ -2,8 +2,16 @@ const test = require('tap').test; const path = require('path'); const unzip = require('../'); -test("get content a docx file without errors", async function () { - const archive = path.join(__dirname, '../testData/office/testfile.docx'); +var test = require('tap').test; +var fs = require('fs'); +var path = require('path'); +var unzip = require('../'); +var il = require('iconv-lite'); +var Promise = require('bluebird'); +var NoopStream = require('../lib/NoopStream'); + +test("get content a docx file without errors", async function (t) { + var archive = path.join(__dirname, '../testData/office/testfile.docx'); const directory = await unzip.Open.file(archive); await Promise.all(directory.files.map(file => file.buffer())); @@ -14,4 +22,18 @@ test("get content a xlsx file without errors", async function () { const directory = await unzip.Open.file(archive); await Promise.all(directory.files.map(file => file.buffer())); +}); + +test("stream retries when the local file header indicates bigger size than central directory", async function (t) { + var archive = path.join(__dirname, '../testData/office/testfile.xlsx'); + let retries = 0, size; + const directory = await unzip.Open.file(archive, {padding: 10}); + const stream = directory.files[0].stream(); + stream.on('streamRetry', _size => { + retries += 1; + size = _size; + }); + await new Promise(resolve => stream.pipe(NoopStream()).on('finish', resolve)); + t.ok(retries === 1, 'retries once'); + t.ok(size > 0, 'size is set'); }); \ No newline at end of file From ca37f3b3c9a6ac4b5e33a4d70e4d9e341851a470 Mon Sep 17 00:00:00 2001 From: Ziggy Jonsson Date: Sat, 11 May 2024 16:08:34 -0400 Subject: [PATCH 5/7] fix eslint --- lib/Open/directory.js | 77 ++++++++++++++++----------------- lib/Open/unzip.js | 99 +++++++++++++++++++++---------------------- test/office-files.js | 15 ++----- 3 files changed, 90 insertions(+), 101 deletions(-) diff --git a/lib/Open/directory.js b/lib/Open/directory.js index 98616b8..97f785c 100644 --- a/lib/Open/directory.js +++ b/lib/Open/directory.js @@ -12,22 +12,21 @@ const signature = Buffer.alloc(4); signature.writeUInt32LE(0x06054b50, 0); async function getCrxHeader(source) { - var sourceStream = source.stream(0).pipe(PullStream()); + const sourceStream = source.stream(0).pipe(PullStream()); let data = await sourceStream.pull(4); - var signature = data.readUInt32LE(0); + const signature = data.readUInt32LE(0); if (signature === 0x34327243) { - var crxHeader; data = await sourceStream.pull(12); - crxHeader = parseBuffer.parse(data, [ + const crxHeader = parseBuffer.parse(data, [ ['version', 4], ['pubKeyLength', 4], ['signatureLength', 4], ]); - + data = await sourceStream.pull(crxHeader.pubKeyLength +crxHeader.signatureLength); - crxHeader.publicKey = data.slice(0,crxHeader.pubKeyLength); + crxHeader.publicKey = data.slice(0, crxHeader.pubKeyLength); crxHeader.signature = data.slice(crxHeader.pubKeyLength); crxHeader.size = 16 + crxHeader.pubKeyLength +crxHeader.signatureLength; return crxHeader; @@ -76,26 +75,24 @@ function parseZip64DirRecord (dir64record) { } module.exports = async function centralDirectory(source, options) { - var endDir = PullStream(), - records = PullStream(), - tailSize = (options && options.tailSize) || 80, - crxHeader, - startOffset, - vars; + const endDir = PullStream(); + const records = PullStream(); + const tailSize = (options && options.tailSize) || 80; + let crxHeader, vars; if (options && options.crx) crxHeader = await getCrxHeader(source); const sourceSize = await source.size(); - source.stream(Math.max(0,sourceSize-tailSize)) - .on('error', function (error) { endDir.emit('error', error) }) + source.stream(Math.max(0, sourceSize-tailSize)) + .on('error', function (error) { endDir.emit('error', error); }) .pipe(endDir); await endDir.pull(signature); - var data = await endDir.pull(22); - startOffset = crxHeader && crxHeader.size || 0; + const data = await endDir.pull(22); + const startOffset = crxHeader && crxHeader.size || 0; vars = parseBuffer.parse(data, [ ['signature', 4], @@ -116,16 +113,16 @@ module.exports = async function centralDirectory(source, options) { vars.offsetToStartOfCentralDirectory == 0xffffffff) { // Offset to zip64 CDL is 20 bytes before normal CDR - const zip64CDLSize = 20 - const zip64CDLOffset = sourceSize - (tailSize - endDir.match + zip64CDLSize) + const zip64CDLSize = 20; + const zip64CDLOffset = sourceSize - (tailSize - endDir.match + zip64CDLSize); const zip64CDLStream = PullStream(); source.stream(zip64CDLOffset).pipe(zip64CDLStream); - const d = await zip64CDLStream.pull(zip64CDLSize) + const d = await zip64CDLStream.pull(zip64CDLSize); const dir64record = await getZip64CentralDirectory(source, d);; - - vars = parseZip64DirRecord(dir64record) + + vars = parseZip64DirRecord(dir64record); } else { vars.offsetToStartOfCentralDirectory += startOffset; @@ -143,32 +140,32 @@ module.exports = async function centralDirectory(source, options) { // make sure path is normalized before using it opts.path = path.resolve(path.normalize(opts.path)); const files = await vars.files; - - return Promise.map(files, function(entry) { + + return Bluebird.map(files, function(entry) { if (entry.type == 'Directory') return; // to avoid zip slip (writing outside of the destination), we resolve // the target path, and make sure it's nested in the intended // destination, or not extract it otherwise. - var extractPath = path.join(opts.path, entry.path); + const extractPath = path.join(opts.path, entry.path); if (extractPath.indexOf(opts.path) != 0) { return; } - var writer = opts.getWriter ? opts.getWriter({path: extractPath}) : Writer({ path: extractPath }); + const writer = opts.getWriter ? opts.getWriter({path: extractPath}) : Writer({ path: extractPath }); return new Promise(function(resolve, reject) { entry.stream(opts.password) - .on('error',reject) + .on('error', reject) .pipe(writer) - .on('close',resolve) - .on('error',reject); + .on('close', resolve) + .on('error', reject); }); }, { concurrency: opts.concurrency > 1 ? opts.concurrency : 1 }); }; - vars.files = Promise.mapSeries(Array(vars.numberOfRecords),async function() { - const data = await records.pull(46) - var vars = vars = parseBuffer.parse(data, [ + vars.files = Bluebird.mapSeries(Array(vars.numberOfRecords), async function() { + const data = await records.pull(46); + const vars = parseBuffer.parse(data, [ ['signature', 4], ['versionMadeBy', 2], ['versionsNeededToExtract', 2], @@ -198,25 +195,25 @@ module.exports = async function centralDirectory(source, options) { const extraField = await records.pull(vars.extraFieldLength); vars.extra = parseExtraField(extraField, vars); - const comment = await records.pull(vars.fileCommentLength); - + const comment = await records.pull(vars.fileCommentLength); + vars.comment = comment; - vars.type = (vars.uncompressedSize === 0 && /[\/\\]$/.test(vars.path)) ? 'Directory' : 'File'; - var padding = options && options.padding || 1000; + vars.type = (vars.uncompressedSize === 0 && /[/\\]$/.test(vars.path)) ? 'Directory' : 'File'; + const padding = options && options.padding || 1000; vars.stream = function(_password) { - var totalSize = 30 + const totalSize = 30 + padding // add an extra buffer - + (vars.extraFieldLength || 0) + + (vars.extraFieldLength || 0) + (vars.fileNameLength || 0) + vars.compressedSize; - return unzip(source, vars.offsetToLocalFileHeader,_password, vars, totalSize); + return unzip(source, vars.offsetToLocalFileHeader, _password, vars, totalSize); }; vars.buffer = function(_password) { return BufferStream(vars.stream(_password)); }; return vars; }); - - return Promise.props(vars); + + return Bluebird.props(vars); }; diff --git a/lib/Open/unzip.js b/lib/Open/unzip.js index 079ca7d..9643c1b 100644 --- a/lib/Open/unzip.js +++ b/lib/Open/unzip.js @@ -7,8 +7,8 @@ const parseDateTime = require('../parseDateTime'); const parseBuffer = require('../parseBuffer'); module.exports = function unzip(source, offset, _password, directoryVars, length, _entry) { - var file = PullStream(), - entry = _entry || Stream.PassThrough(); + const file = PullStream(); + const entry = _entry || Stream.PassThrough(); const req = source.stream(offset, length); req.pipe(file).on('error', function(e) { @@ -34,12 +34,12 @@ module.exports = function unzip(source, offset, _password, directoryVars, length ['extraFieldLength', 2], ]); - var localSize = 30 + const localSize = 30 + 100 // add extra padding - + (vars.extraFieldLength || 0) + + (vars.extraFieldLength || 0) + (vars.fileNameLength || 0) + vars.compressedSize; - + if (localSize > length) { entry.emit('streamRetry', localSize); return unzip(source, offset, _password, directoryVars, localSize, entry); @@ -48,76 +48,75 @@ module.exports = function unzip(source, offset, _password, directoryVars, length vars.lastModifiedDateTime = parseDateTime(vars.lastModifiedDate, vars.lastModifiedTime); const fileName = await file.pull(vars.fileNameLength); - + vars.fileName = fileName.toString('utf8'); const extraField = await file.pull(vars.extraFieldLength); - - var checkEncryption; + vars.extra = parseExtraField(extraField, vars); // Ignore logal file header vars if the directory vars are available if (directoryVars && directoryVars.compressedSize) vars = directoryVars; if (vars.flags & 0x01) { - const header = await file.pull(12) - + const header = await file.pull(12); + if (!_password) throw new Error('MISSING_PASSWORD'); - var decrypt = Decrypt(); + const decrypt = Decrypt(); String(_password).split('').forEach(function(d) { decrypt.update(d); }); - for (var i=0; i < header.length; i++) + for (let i=0; i < header.length; i++) header[i] = decrypt.decryptByte(header[i]); vars.decrypt = decrypt; vars.compressedSize -= 12; - var check = (vars.flags & 0x8) ? (vars.lastModifiedTime >> 8) & 0xff : (vars.crc32 >> 24) & 0xff; + const check = (vars.flags & 0x8) ? (vars.lastModifiedTime >> 8) & 0xff : (vars.crc32 >> 24) & 0xff; if (header[11] !== check) throw new Error('BAD_PASSWORD'); }; - entry.emit('vars',vars); - - var fileSizeKnown = !(vars.flags & 0x08) || vars.compressedSize > 0, - eof; - - const inflater = vars.compressionMethod ? zlib.createInflateRaw() : Stream.PassThrough(); - - if (fileSizeKnown) { - entry.size = vars.uncompressedSize; - eof = vars.compressedSize; - } else { - eof = Buffer.alloc(4); - eof.writeUInt32LE(0x08074b50, 0); - } - - let stream = file.stream(eof); - - if (vars.decrypt) - stream = stream.pipe(vars.decrypt.stream()); - - stream - .pipe(inflater) - .on('error', function(err) { entry.emit('error', err);}) - .pipe(entry) - .on('finish', function() { - if(req.destroy) - req.destroy(); - else if (req.abort) - req.abort(); - else if (req.close) - req.close(); - else if (req.push) - req.push(); - else - console.log('warning - unable to close stream'); - }); - }) + entry.emit('vars', vars); + + const fileSizeKnown = !(vars.flags & 0x08) || vars.compressedSize > 0; + let eof; + + const inflater = vars.compressionMethod ? zlib.createInflateRaw() : Stream.PassThrough(); + + if (fileSizeKnown) { + entry.size = vars.uncompressedSize; + eof = vars.compressedSize; + } else { + eof = Buffer.alloc(4); + eof.writeUInt32LE(0x08074b50, 0); + } + + let stream = file.stream(eof); + + if (vars.decrypt) + stream = stream.pipe(vars.decrypt.stream()); + + stream + .pipe(inflater) + .on('error', function(err) { entry.emit('error', err);}) + .pipe(entry) + .on('finish', function() { + if(req.destroy) + req.destroy(); + else if (req.abort) + req.abort(); + else if (req.close) + req.close(); + else if (req.push) + req.push(); + else + console.log('warning - unable to close stream'); + }); + }) .catch(function(e) { entry.emit('error', e); }); diff --git a/test/office-files.js b/test/office-files.js index 00068e2..b0a9fe4 100644 --- a/test/office-files.js +++ b/test/office-files.js @@ -1,17 +1,10 @@ const test = require('tap').test; const path = require('path'); const unzip = require('../'); +const NoopStream = require('../lib/NoopStream'); -var test = require('tap').test; -var fs = require('fs'); -var path = require('path'); -var unzip = require('../'); -var il = require('iconv-lite'); -var Promise = require('bluebird'); -var NoopStream = require('../lib/NoopStream'); - -test("get content a docx file without errors", async function (t) { - var archive = path.join(__dirname, '../testData/office/testfile.docx'); +test("get content a docx file without errors", async function () { + const archive = path.join(__dirname, '../testData/office/testfile.docx'); const directory = await unzip.Open.file(archive); await Promise.all(directory.files.map(file => file.buffer())); @@ -25,7 +18,7 @@ test("get content a xlsx file without errors", async function () { }); test("stream retries when the local file header indicates bigger size than central directory", async function (t) { - var archive = path.join(__dirname, '../testData/office/testfile.xlsx'); + const archive = path.join(__dirname, '../testData/office/testfile.xlsx'); let retries = 0, size; const directory = await unzip.Open.file(archive, {padding: 10}); const stream = directory.files[0].stream(); From 9ef3ade92a511282a761f3f16ab6d34a93170e83 Mon Sep 17 00:00:00 2001 From: Ziggy Jonsson Date: Sat, 11 May 2024 18:49:26 -0400 Subject: [PATCH 6/7] wip --- jsconfig.json | 13 ++ lib/BufferStream.js | 2 +- lib/Open/directory.js | 10 +- lib/Open/index.js | 2 +- lib/Open/unzip.js | 6 +- lib/PullStream.js | 220 ++++++++++--------- lib/parse.js | 490 +++++++++++++++++++++--------------------- package.json | 2 + 8 files changed, 378 insertions(+), 367 deletions(-) create mode 100644 jsconfig.json diff --git a/jsconfig.json b/jsconfig.json new file mode 100644 index 0000000..1c2fef5 --- /dev/null +++ b/jsconfig.json @@ -0,0 +1,13 @@ +{ + "compilerOptions": { + "checkJs": true, + "target": "ES2022", + "moduleResolution":"node", + "types": ["node"] + }, + "exclude": [ + "node_modules", + "test", + "coverage" + ] +} \ No newline at end of file diff --git a/lib/BufferStream.js b/lib/BufferStream.js index 0ee7e5e..5c62ac7 100644 --- a/lib/BufferStream.js +++ b/lib/BufferStream.js @@ -3,7 +3,7 @@ const Stream = require('stream'); module.exports = function(entry) { return new Promise(function(resolve, reject) { const chunks = []; - const bufferStream = Stream.Transform() + const bufferStream = new Stream.Transform() .on('finish', function() { resolve(Buffer.concat(chunks)); }) diff --git a/lib/Open/directory.js b/lib/Open/directory.js index 97f785c..a8876ae 100644 --- a/lib/Open/directory.js +++ b/lib/Open/directory.js @@ -12,7 +12,7 @@ const signature = Buffer.alloc(4); signature.writeUInt32LE(0x06054b50, 0); async function getCrxHeader(source) { - const sourceStream = source.stream(0).pipe(PullStream()); + const sourceStream = source.stream(0).pipe(new PullStream()); let data = await sourceStream.pull(4); const signature = data.readUInt32LE(0); @@ -46,7 +46,7 @@ function getZip64CentralDirectory(source, zip64CDL) { throw new Error('invalid zip64 end of central dir locator signature (0x07064b50): 0x' + d64loc.signature.toString(16)); } - const dir64 = PullStream(); + const dir64 = new PullStream(); source.stream(d64loc.offsetToStartOfCentralDirectory).pipe(dir64); return dir64.pull(56); @@ -75,8 +75,8 @@ function parseZip64DirRecord (dir64record) { } module.exports = async function centralDirectory(source, options) { - const endDir = PullStream(); - const records = PullStream(); + const endDir = new PullStream(); + const records = new PullStream(); const tailSize = (options && options.tailSize) || 80; let crxHeader, vars; @@ -115,7 +115,7 @@ module.exports = async function centralDirectory(source, options) { // Offset to zip64 CDL is 20 bytes before normal CDR const zip64CDLSize = 20; const zip64CDLOffset = sourceSize - (tailSize - endDir.match + zip64CDLSize); - const zip64CDLStream = PullStream(); + const zip64CDLStream = new PullStream(); source.stream(zip64CDLOffset).pipe(zip64CDLStream); diff --git a/lib/Open/index.js b/lib/Open/index.js index 83c349b..6675508 100644 --- a/lib/Open/index.js +++ b/lib/Open/index.js @@ -6,7 +6,7 @@ module.exports = { buffer: function(buffer, options) { const source = { stream: function(offset, length) { - const stream = Stream.PassThrough(); + const stream = new Stream.PassThrough(); const end = length ? offset + length : undefined; stream.end(buffer.slice(offset, end)); return stream; diff --git a/lib/Open/unzip.js b/lib/Open/unzip.js index 9643c1b..d9e09ec 100644 --- a/lib/Open/unzip.js +++ b/lib/Open/unzip.js @@ -7,8 +7,8 @@ const parseDateTime = require('../parseDateTime'); const parseBuffer = require('../parseBuffer'); module.exports = function unzip(source, offset, _password, directoryVars, length, _entry) { - const file = PullStream(); - const entry = _entry || Stream.PassThrough(); + const file = new PullStream(); + const entry = _entry || new Stream.PassThrough(); const req = source.stream(offset, length); req.pipe(file).on('error', function(e) { @@ -85,7 +85,7 @@ module.exports = function unzip(source, offset, _password, directoryVars, length const fileSizeKnown = !(vars.flags & 0x08) || vars.compressedSize > 0; let eof; - const inflater = vars.compressionMethod ? zlib.createInflateRaw() : Stream.PassThrough(); + const inflater = vars.compressionMethod ? zlib.createInflateRaw() : new Stream.PassThrough(); if (fileSizeKnown) { entry.size = vars.uncompressedSize; diff --git a/lib/PullStream.js b/lib/PullStream.js index 93f18e4..433d29c 100644 --- a/lib/PullStream.js +++ b/lib/PullStream.js @@ -1,139 +1,137 @@ const Stream = require('stream'); -const util = require('util'); const strFunction = 'function'; -function PullStream() { - if (!(this instanceof PullStream)) - return new PullStream(); - - Stream.Duplex.call(this, {decodeStrings:false, objectMode:true}); - this.buffer = Buffer.from(''); - const self = this; - self.on('finish', function() { - self.finished = true; - self.emit('chunk', false); - }); -} - -util.inherits(PullStream, Stream.Duplex); +class PullStream extends Stream.Duplex { + finished; + match; + __emittedError; + constructor(opts) { + super({decodeStrings:false, objectMode:true}); + this._opts = opts; + this.buffer = Buffer.from(''); + this.on('finish', () => { + this.finished = true; + this.emit('chunk', false); + }); + } -PullStream.prototype._write = function(chunk, e, cb) { - this.buffer = Buffer.concat([this.buffer, chunk]); - this.cb = cb; - this.emit('chunk'); -}; + _write(chunk, e, cb) { + this.buffer = Buffer.concat([this.buffer, chunk]); + this.cb = cb; + this.emit('chunk'); + }; -// The `eof` parameter is interpreted as `file_length` if the type is number -// otherwise (i.e. buffer) it is interpreted as a pattern signaling end of stream -PullStream.prototype.stream = function(eof, includeEof) { - const p = Stream.PassThrough(); - let done; - const self= this; + // The `eof` parameter is interpreted as `file_length` if the type is number + // otherwise (i.e. buffer) it is interpreted as a pattern signaling end of stream + stream(eof, includeEof) { + const p = new Stream.PassThrough(); + let done; + const self= this; - function cb() { - if (typeof self.cb === strFunction) { - const callback = self.cb; - self.cb = undefined; - return callback(); + function cb() { + if (typeof self.cb === strFunction) { + const callback = self.cb; + self.cb = undefined; + return callback(); + } } - } - function pull() { - let packet; - if (self.buffer && self.buffer.length) { - if (typeof eof === 'number') { - packet = self.buffer.slice(0, eof); - self.buffer = self.buffer.slice(eof); - eof -= packet.length; - done = done || !eof; - } else { - let match = self.buffer.indexOf(eof); - if (match !== -1) { + function pull() { + let packet; + if (self.buffer && self.buffer.length) { + if (typeof eof === 'number') { + packet = self.buffer.slice(0, eof); + self.buffer = self.buffer.slice(eof); + eof -= packet.length; + done = done || !eof; + } else { + let match = self.buffer.indexOf(eof); + if (match !== -1) { // store signature match byte offset to allow us to reference // this for zip64 offset - self.match = match; - if (includeEof) match = match + eof.length; - packet = self.buffer.slice(0, match); - self.buffer = self.buffer.slice(match); - done = true; - } else { - const len = self.buffer.length - eof.length; - if (len <= 0) { - cb(); + self.match = match; + if (includeEof) match = match + eof.length; + packet = self.buffer.slice(0, match); + self.buffer = self.buffer.slice(match); + done = true; } else { - packet = self.buffer.slice(0, len); - self.buffer = self.buffer.slice(len); + const len = self.buffer.length - eof.length; + if (len <= 0) { + cb(); + } else { + packet = self.buffer.slice(0, len); + self.buffer = self.buffer.slice(len); + } } } + if (packet) p.write(packet, function() { + if (self.buffer.length === 0 || (eof.length && self.buffer.length <= eof.length)) cb(); + }); } - if (packet) p.write(packet, function() { - if (self.buffer.length === 0 || (eof.length && self.buffer.length <= eof.length)) cb(); - }); - } - if (!done) { - if (self.finished) { + if (!done) { + if (self.finished) { + self.removeListener('chunk', pull); + self.emit('error', new Error('FILE_ENDED')); + return; + } + + } else { self.removeListener('chunk', pull); - self.emit('error', new Error('FILE_ENDED')); - return; + p.end(); } - - } else { - self.removeListener('chunk', pull); - p.end(); } - } - self.on('chunk', pull); - pull(); - return p; -}; - -PullStream.prototype.pull = function(eof, includeEof) { - if (eof === 0) return Promise.resolve(''); + self.on('chunk', pull); + pull(); + return p; + }; - // If we already have the required data in buffer - // we can resolve the request immediately - if (!isNaN(eof) && this.buffer.length > eof) { - const data = this.buffer.slice(0, eof); - this.buffer = this.buffer.slice(eof); - return Promise.resolve(data); - } + pull(eof, includeEof) { + if (eof === 0) return Promise.resolve(''); - // Otherwise we stream until we have it - let buffer = Buffer.from(''); - const self = this; + // If we already have the required data in buffer + // we can resolve the request immediately + if (!isNaN(eof) && this.buffer.length > eof) { + const data = this.buffer.slice(0, eof); + this.buffer = this.buffer.slice(eof); + return Promise.resolve(data); + } - const concatStream = new Stream.Transform(); - concatStream._transform = function(d, e, cb) { - buffer = Buffer.concat([buffer, d]); - cb(); - }; + // Otherwise we stream until we have it + let buffer = Buffer.from(''); + const self = this; - let rejectHandler; - let pullStreamRejectHandler; - return new Promise(function(resolve, reject) { - rejectHandler = reject; - pullStreamRejectHandler = function(e) { - self.__emittedError = e; - reject(e); + const concatStream = new Stream.Transform(); + concatStream._transform = function(d, e, cb) { + buffer = Buffer.concat([buffer, d]); + cb(); }; - if (self.finished) - return reject(new Error('FILE_ENDED')); - self.once('error', pullStreamRejectHandler); // reject any errors from pullstream itself - self.stream(eof, includeEof) - .on('error', reject) - .pipe(concatStream) - .on('finish', function() {resolve(buffer);}) - .on('error', reject); - }) - .finally(function() { - self.removeListener('error', rejectHandler); - self.removeListener('error', pullStreamRejectHandler); - }); -}; -PullStream.prototype._read = function(){}; + let rejectHandler; + let pullStreamRejectHandler; + return new Promise(function(resolve, reject) { + rejectHandler = reject; + pullStreamRejectHandler = function(e) { + self.__emittedError = e; + reject(e); + }; + if (self.finished) + return reject(new Error('FILE_ENDED')); + self.once('error', pullStreamRejectHandler); // reject any errors from pullstream itself + self.stream(eof, includeEof) + .on('error', reject) + .pipe(concatStream) + .on('finish', function() {resolve(buffer);}) + .on('error', reject); + }) + .finally(function() { + self.removeListener('error', rejectHandler); + self.removeListener('error', pullStreamRejectHandler); + }); + }; + _read(){}; +} module.exports = PullStream; diff --git a/lib/parse.js b/lib/parse.js index 0921f4a..1b35cf7 100644 --- a/lib/parse.js +++ b/lib/parse.js @@ -1,4 +1,3 @@ -const util = require('util'); const zlib = require('zlib'); const Stream = require('stream'); const PullStream = require('./PullStream'); @@ -12,277 +11,276 @@ const parseBuffer = require('./parseBuffer'); const endDirectorySignature = Buffer.alloc(4); endDirectorySignature.writeUInt32LE(0x06054b50, 0); -function Parse(opts) { - if (!(this instanceof Parse)) { - return new Parse(opts); +class Parse extends PullStream { + constructor(opts) { + super(opts || { verbose: false }); + const self = this; + + self.on('finish', function() { + self.emit('end'); + self.emit('close'); + }); + self._readRecord().catch(function(e) { + if (!self.__emittedError || self.__emittedError !== e) + self.emit('error', e); + }); } - const self = this; - self._opts = opts || { verbose: false }; - - PullStream.call(self, self._opts); - self.on('finish', function() { - self.emit('end'); - self.emit('close'); - }); - self._readRecord().catch(function(e) { - if (!self.__emittedError || self.__emittedError !== e) - self.emit('error', e); - }); -} -util.inherits(Parse, PullStream); - -Parse.prototype._readRecord = function () { - const self = this; - - return self.pull(4).then(function(data) { - if (data.length === 0) - return; - - const signature = data.readUInt32LE(0); - - if (signature === 0x34327243) { - return self._readCrxHeader(); - } - if (signature === 0x04034b50) { - return self._readFile(); - } - else if (signature === 0x02014b50) { - self.reachedCD = true; - return self._readCentralDirectoryFileHeader(); - } - else if (signature === 0x06054b50) { - return self._readEndOfCentralDirectoryRecord(); - } - else if (self.reachedCD) { + + _readRecord() { + const self = this; + + return self.pull(4).then(function(data) { + if (data.length === 0) + return; + + const signature = data.readUInt32LE(0); + + if (signature === 0x34327243) { + return self._readCrxHeader(); + } + if (signature === 0x04034b50) { + return self._readFile(); + } + else if (signature === 0x02014b50) { + self.reachedCD = true; + return self._readCentralDirectoryFileHeader(); + } + else if (signature === 0x06054b50) { + return self._readEndOfCentralDirectoryRecord(); + } + else if (self.reachedCD) { // _readEndOfCentralDirectoryRecord expects the EOCD // signature to be consumed so set includeEof=true - const includeEof = true; - return self.pull(endDirectorySignature, includeEof).then(function() { - return self._readEndOfCentralDirectoryRecord(); - }); - } - else - self.emit('error', new Error('invalid signature: 0x' + signature.toString(16))); - }).then((function(loop) { - if(loop) { - return self._readRecord(); - } - })); -}; + const includeEof = true; + return self.pull(endDirectorySignature, includeEof).then(function() { + return self._readEndOfCentralDirectoryRecord(); + }); + } + else + self.emit('error', new Error('invalid signature: 0x' + signature.toString(16))); + }).then((function(loop) { + if(loop) { + return self._readRecord(); + } + })); + }; + + _readCrxHeader() { + const self = this; + return self.pull(12).then(function(data) { + self.crxHeader = parseBuffer.parse(data, [ + ['version', 4], + ['pubKeyLength', 4], + ['signatureLength', 4], + ]); + return self.pull(self.crxHeader.pubKeyLength + self.crxHeader.signatureLength); + }).then(function(data) { + self.crxHeader.publicKey = data.slice(0, self.crxHeader.pubKeyLength); + self.crxHeader.signature = data.slice(self.crxHeader.pubKeyLength); + self.emit('crx-header', self.crxHeader); + return true; + }); + }; + + _readFile() { + const self = this; + return self.pull(26).then(function(data) { + const vars = parseBuffer.parse(data, [ + ['versionsNeededToExtract', 2], + ['flags', 2], + ['compressionMethod', 2], + ['lastModifiedTime', 2], + ['lastModifiedDate', 2], + ['crc32', 4], + ['compressedSize', 4], + ['uncompressedSize', 4], + ['fileNameLength', 2], + ['extraFieldLength', 2], + ]); + + vars.lastModifiedDateTime = parseDateTime(vars.lastModifiedDate, vars.lastModifiedTime); + + if (self.crxHeader) vars.crxHeader = self.crxHeader; + + return self.pull(vars.fileNameLength).then(function(fileNameBuffer) { + const fileName = fileNameBuffer.toString('utf8'); + const entry = Stream.PassThrough(); + let __autodraining = false; + + entry.autodrain = function() { + __autodraining = true; + const draining = entry.pipe(NoopStream()); + draining.promise = function() { + return new Promise(function(resolve, reject) { + draining.on('finish', resolve); + draining.on('error', reject); + }); + }; + return draining; + }; -Parse.prototype._readCrxHeader = function() { - const self = this; - return self.pull(12).then(function(data) { - self.crxHeader = parseBuffer.parse(data, [ - ['version', 4], - ['pubKeyLength', 4], - ['signatureLength', 4], - ]); - return self.pull(self.crxHeader.pubKeyLength + self.crxHeader.signatureLength); - }).then(function(data) { - self.crxHeader.publicKey = data.slice(0, self.crxHeader.pubKeyLength); - self.crxHeader.signature = data.slice(self.crxHeader.pubKeyLength); - self.emit('crx-header', self.crxHeader); - return true; - }); -}; + entry.buffer = function() { + return BufferStream(entry); + }; -Parse.prototype._readFile = function () { - const self = this; - return self.pull(26).then(function(data) { - const vars = parseBuffer.parse(data, [ - ['versionsNeededToExtract', 2], - ['flags', 2], - ['compressionMethod', 2], - ['lastModifiedTime', 2], - ['lastModifiedDate', 2], - ['crc32', 4], - ['compressedSize', 4], - ['uncompressedSize', 4], - ['fileNameLength', 2], - ['extraFieldLength', 2], - ]); - - vars.lastModifiedDateTime = parseDateTime(vars.lastModifiedDate, vars.lastModifiedTime); - - if (self.crxHeader) vars.crxHeader = self.crxHeader; - - return self.pull(vars.fileNameLength).then(function(fileNameBuffer) { - const fileName = fileNameBuffer.toString('utf8'); - const entry = Stream.PassThrough(); - let __autodraining = false; - - entry.autodrain = function() { - __autodraining = true; - const draining = entry.pipe(NoopStream()); - draining.promise = function() { - return new Promise(function(resolve, reject) { - draining.on('finish', resolve); - draining.on('error', reject); - }); + entry.path = fileName; + entry.props = {}; + entry.props.path = fileName; + entry.props.pathBuffer = fileNameBuffer; + entry.props.flags = { + "isUnicode": (vars.flags & 0x800) != 0 }; - return draining; - }; - - entry.buffer = function() { - return BufferStream(entry); - }; - - entry.path = fileName; - entry.props = {}; - entry.props.path = fileName; - entry.props.pathBuffer = fileNameBuffer; - entry.props.flags = { - "isUnicode": (vars.flags & 0x800) != 0 - }; - entry.type = (vars.uncompressedSize === 0 && /[/\\]$/.test(fileName)) ? 'Directory' : 'File'; - - if (self._opts.verbose) { - if (entry.type === 'Directory') { - console.log(' creating:', fileName); - } else if (entry.type === 'File') { - if (vars.compressionMethod === 0) { - console.log(' extracting:', fileName); - } else { - console.log(' inflating:', fileName); + entry.type = (vars.uncompressedSize === 0 && /[/\\]$/.test(fileName)) ? 'Directory' : 'File'; + + if (self._opts.verbose) { + if (entry.type === 'Directory') { + console.log(' creating:', fileName); + } else if (entry.type === 'File') { + if (vars.compressionMethod === 0) { + console.log(' extracting:', fileName); + } else { + console.log(' inflating:', fileName); + } } } - } - - return self.pull(vars.extraFieldLength).then(function(extraField) { - const extra = parseExtraField(extraField, vars); - entry.vars = vars; - entry.extra = extra; + return self.pull(vars.extraFieldLength).then(function(extraField) { + const extra = parseExtraField(extraField, vars); - if (self._opts.forceStream) { - self.push(entry); - } else { - self.emit('entry', entry); + entry.vars = vars; + entry.extra = extra; - if (self._readableState.pipesCount || (self._readableState.pipes && self._readableState.pipes.length)) + if (self._opts.forceStream) { self.push(entry); - } + } else { + self.emit('entry', entry); - if (self._opts.verbose) - console.log({ - filename:fileName, - vars: vars, - extra: extra - }); + if (self._readableState.pipesCount || (self._readableState.pipes && self._readableState.pipes.length)) + self.push(entry); + } - const fileSizeKnown = !(vars.flags & 0x08) || vars.compressedSize > 0; - let eof; + if (self._opts.verbose) + console.log({ + filename:fileName, + vars: vars, + extra: extra + }); - entry.__autodraining = __autodraining; // expose __autodraining for test purposes - const inflater = (vars.compressionMethod && !__autodraining) ? zlib.createInflateRaw() : Stream.PassThrough(); + const fileSizeKnown = !(vars.flags & 0x08) || vars.compressedSize > 0; + let eof; - if (fileSizeKnown) { - entry.size = vars.uncompressedSize; - eof = vars.compressedSize; - } else { - eof = Buffer.alloc(4); - eof.writeUInt32LE(0x08074b50, 0); - } + entry.__autodraining = __autodraining; // expose __autodraining for test purposes + const inflater = (vars.compressionMethod && !__autodraining) ? zlib.createInflateRaw() : Stream.PassThrough(); - return new Promise(function(resolve, reject) { - pipeline( - self.stream(eof), - inflater, - entry, - function (err) { - if (err) { - return reject(err); - } + if (fileSizeKnown) { + entry.size = vars.uncompressedSize; + eof = vars.compressedSize; + } else { + eof = Buffer.alloc(4); + eof.writeUInt32LE(0x08074b50, 0); + } - return fileSizeKnown ? resolve(fileSizeKnown) : self._processDataDescriptor(entry).then(resolve).catch(reject); - } - ); + return new Promise(function(resolve, reject) { + pipeline( + self.stream(eof), + inflater, + entry, + function (err) { + if (err) { + return reject(err); + } + + return fileSizeKnown ? resolve(fileSizeKnown) : self._processDataDescriptor(entry).then(resolve).catch(reject); + } + ); + }); }); }); }); - }); -}; - -Parse.prototype._processDataDescriptor = function (entry) { - const self = this; - return self.pull(16).then(function(data) { - const vars = parseBuffer.parse(data, [ - ['dataDescriptorSignature', 4], - ['crc32', 4], - ['compressedSize', 4], - ['uncompressedSize', 4], - ]); - - entry.size = vars.uncompressedSize; - return true; - }); -}; - -Parse.prototype._readCentralDirectoryFileHeader = function () { - const self = this; - return self.pull(42).then(function(data) { - const vars = parseBuffer.parse(data, [ - ['versionMadeBy', 2], - ['versionsNeededToExtract', 2], - ['flags', 2], - ['compressionMethod', 2], - ['lastModifiedTime', 2], - ['lastModifiedDate', 2], - ['crc32', 4], - ['compressedSize', 4], - ['uncompressedSize', 4], - ['fileNameLength', 2], - ['extraFieldLength', 2], - ['fileCommentLength', 2], - ['diskNumber', 2], - ['internalFileAttributes', 2], - ['externalFileAttributes', 4], - ['offsetToLocalFileHeader', 4], - ]); - - return self.pull(vars.fileNameLength).then(function(fileName) { - vars.fileName = fileName.toString('utf8'); - return self.pull(vars.extraFieldLength); - }) - .then(function() { - return self.pull(vars.fileCommentLength); + }; + + _processDataDescriptor(entry) { + const self = this; + return self.pull(16).then(function(data) { + const vars = parseBuffer.parse(data, [ + ['dataDescriptorSignature', 4], + ['crc32', 4], + ['compressedSize', 4], + ['uncompressedSize', 4], + ]); + + entry.size = vars.uncompressedSize; + return true; + }); + }; + + _readCentralDirectoryFileHeader() { + const self = this; + return self.pull(42).then(function(data) { + const vars = parseBuffer.parse(data, [ + ['versionMadeBy', 2], + ['versionsNeededToExtract', 2], + ['flags', 2], + ['compressionMethod', 2], + ['lastModifiedTime', 2], + ['lastModifiedDate', 2], + ['crc32', 4], + ['compressedSize', 4], + ['uncompressedSize', 4], + ['fileNameLength', 2], + ['extraFieldLength', 2], + ['fileCommentLength', 2], + ['diskNumber', 2], + ['internalFileAttributes', 2], + ['externalFileAttributes', 4], + ['offsetToLocalFileHeader', 4], + ]); + + return self.pull(vars.fileNameLength).then(function(fileName) { + vars.fileName = fileName.toString('utf8'); + return self.pull(vars.extraFieldLength); }) - .then(function() { - return true; + .then(function() { + return self.pull(vars.fileCommentLength); + }) + .then(function() { + return true; + }); + }); + }; + + _readEndOfCentralDirectoryRecord() { + const self = this; + return self.pull(18).then(function(data) { + + const vars = parseBuffer.parse(data, [ + ['diskNumber', 2], + ['diskStart', 2], + ['numberOfRecordsOnDisk', 2], + ['numberOfRecords', 2], + ['sizeOfCentralDirectory', 4], + ['offsetToStartOfCentralDirectory', 4], + ['commentLength', 2], + ]); + + return self.pull(vars.commentLength).then(function() { + self.end(); + self.push(null); }); - }); -}; -Parse.prototype._readEndOfCentralDirectoryRecord = function() { - const self = this; - return self.pull(18).then(function(data) { - - const vars = parseBuffer.parse(data, [ - ['diskNumber', 2], - ['diskStart', 2], - ['numberOfRecordsOnDisk', 2], - ['numberOfRecords', 2], - ['sizeOfCentralDirectory', 4], - ['offsetToStartOfCentralDirectory', 4], - ['commentLength', 2], - ]); - - return self.pull(vars.commentLength).then(function() { - self.end(); - self.push(null); }); + }; - }); -}; + promise() { + const self = this; + return new Promise(function(resolve, reject) { + self.on('finish', resolve); + self.on('error', reject); + }); + }; +} -Parse.prototype.promise = function() { - const self = this; - return new Promise(function(resolve, reject) { - self.on('finish', resolve); - self.on('error', reject); - }); +module.exports = function(opts) { + return new Parse(opts); }; - -module.exports = Parse; diff --git a/package.json b/package.json index 071a492..9f20cf8 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,8 @@ }, "devDependencies": { "@eslint/js": "^9.2.0", + "@types/node": "^20.12.11", + "@types/bluebird": "^3.5.42", "aws-sdk": "^2.77.0", "dirdiff": ">= 0.0.1 < 1", "eslint": "^9.2.0", From 8c16d586114dc9156386f0b4bdc6c42d7a0e31fb Mon Sep 17 00:00:00 2001 From: Ziggy Jonsson Date: Sun, 12 May 2024 09:15:19 -0400 Subject: [PATCH 7/7] fix js errors --- .github/workflows/test.yml | 1 + .gitignore | 1 - jsconfig.json | 4 ++-- lib/Decrypt.js | 7 +++++-- lib/NoopStream.js | 18 ++++++++---------- lib/extract.js | 1 + lib/parse.js | 30 ++++++++++++++++++++---------- lib/parseExtraField.js | 4 +--- lib/parseOne.js | 6 +++--- package.json | 3 ++- test/compressed-crx.js | 8 +++++--- test/office-files.js | 4 ++-- test/streamSingleEntry.js | 2 +- test/uncompressed.js | 4 ++-- 14 files changed, 53 insertions(+), 40 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index eefdaf4..1fd3b41 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,6 +19,7 @@ jobs: node-version: 18.x - run: npm install - run: npx eslint . + - run: npx tsc -p jsconfig.json test: runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index ed57018..97223ba 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,5 @@ /.idea /node_modules -/test.js /.nyc_output/ /coverage/ .tap/ diff --git a/jsconfig.json b/jsconfig.json index 1c2fef5..e7d635f 100644 --- a/jsconfig.json +++ b/jsconfig.json @@ -3,11 +3,11 @@ "checkJs": true, "target": "ES2022", "moduleResolution":"node", - "types": ["node"] + "types": ["node"], + "maxNodeModuleJsDepth": 0, }, "exclude": [ "node_modules", - "test", "coverage" ] } \ No newline at end of file diff --git a/lib/Decrypt.js b/lib/Decrypt.js index 447d8af..ec4fb0a 100644 --- a/lib/Decrypt.js +++ b/lib/Decrypt.js @@ -21,7 +21,7 @@ function crc(ch, crc) { if (ch.charCodeAt) ch = ch.charCodeAt(0); - + //@ts-ignore return (bigInt(crc).shiftRight(8).and(0xffffff)).xor(table[bigInt(crc).xor(ch).and(0xff)]).value; } @@ -36,7 +36,9 @@ function Decrypt() { Decrypt.prototype.update = function(h) { this.key0 = crc(h, this.key0); + //@ts-ignore this.key1 = bigInt(this.key0).and(255).and(4294967295).add(this.key1); + //@ts-ignore this.key1 = bigInt(this.key1).multiply(134775813).add(1).and(4294967295).value; this.key2 = crc(bigInt(this.key1).shiftRight(24).and(255), this.key2); }; @@ -44,13 +46,14 @@ Decrypt.prototype.update = function(h) { Decrypt.prototype.decryptByte = function(c) { const k = bigInt(this.key2).or(2); + //@ts-ignore c = c ^ bigInt(k).multiply(bigInt(k^1)).shiftRight(8).and(255); this.update(c); return c; }; Decrypt.prototype.stream = function() { - const stream = Stream.Transform(), + const stream = new Stream.Transform(), self = this; stream._transform = function(d, e, cb) { diff --git a/lib/NoopStream.js b/lib/NoopStream.js index 98c05f8..d28dfc2 100644 --- a/lib/NoopStream.js +++ b/lib/NoopStream.js @@ -1,14 +1,12 @@ const Stream = require('stream'); -const util = require('util'); -function NoopStream() { - if (!(this instanceof NoopStream)) { - return new NoopStream(); - } - Stream.Transform.call(this); +class NoopStream extends Stream.Transform { + _transform(d, e, cb) { cb() ;}; + promise() { + return new Promise((resolve, reject) => { + this.on('finish', resolve); + this.on('error', reject); + }); + }; } -util.inherits(NoopStream, Stream.Transform); - -NoopStream.prototype._transform = function(d, e, cb) { cb() ;}; - module.exports = NoopStream; \ No newline at end of file diff --git a/lib/extract.js b/lib/extract.js index 31d725a..8100c34 100644 --- a/lib/extract.js +++ b/lib/extract.js @@ -1,3 +1,4 @@ +//@ts-nocheck module.exports = Extract; const Parse = require('./parse'); diff --git a/lib/parse.js b/lib/parse.js index 1b35cf7..4f14015 100644 --- a/lib/parse.js +++ b/lib/parse.js @@ -11,7 +11,23 @@ const parseBuffer = require('./parseBuffer'); const endDirectorySignature = Buffer.alloc(4); endDirectorySignature.writeUInt32LE(0x06054b50, 0); +class Entry extends Stream.PassThrough{ + autodrain; + buffer; + path; + props; + CSSFontFeatureValuesRule; + type; + extra; + size; + vars; + __autodraining; +} + + class Parse extends PullStream { + reachedCD; + crxHeader; constructor(opts) { super(opts || { verbose: false }); const self = this; @@ -105,18 +121,12 @@ class Parse extends PullStream { return self.pull(vars.fileNameLength).then(function(fileNameBuffer) { const fileName = fileNameBuffer.toString('utf8'); - const entry = Stream.PassThrough(); + const entry = new Entry(); let __autodraining = false; entry.autodrain = function() { __autodraining = true; - const draining = entry.pipe(NoopStream()); - draining.promise = function() { - return new Promise(function(resolve, reject) { - draining.on('finish', resolve); - draining.on('error', reject); - }); - }; + const draining = entry.pipe(new NoopStream()); return draining; }; @@ -155,7 +165,7 @@ class Parse extends PullStream { self.push(entry); } else { self.emit('entry', entry); - + //@ts-ignore if (self._readableState.pipesCount || (self._readableState.pipes && self._readableState.pipes.length)) self.push(entry); } @@ -171,7 +181,7 @@ class Parse extends PullStream { let eof; entry.__autodraining = __autodraining; // expose __autodraining for test purposes - const inflater = (vars.compressionMethod && !__autodraining) ? zlib.createInflateRaw() : Stream.PassThrough(); + const inflater = (vars.compressionMethod && !__autodraining) ? zlib.createInflateRaw() : new Stream.PassThrough(); if (fileSizeKnown) { entry.size = vars.uncompressedSize; diff --git a/lib/parseExtraField.js b/lib/parseExtraField.js index 42d178f..6f4b16a 100644 --- a/lib/parseExtraField.js +++ b/lib/parseExtraField.js @@ -1,7 +1,7 @@ const parseBuffer = require('./parseBuffer'); module.exports = function(extraField, vars) { - let extra; + let extra = {}; // Find the ZIP64 header, if present. while(!extra && extraField && extraField.length) { const candidateExtra = parseBuffer.parse(extraField, [ @@ -22,8 +22,6 @@ module.exports = function(extraField, vars) { } } - extra = extra || {}; - if (vars.compressedSize === 0xffffffff) vars.compressedSize = extra.compressedSize; diff --git a/lib/parseOne.js b/lib/parseOne.js index 275dde0..322220a 100644 --- a/lib/parseOne.js +++ b/lib/parseOne.js @@ -4,9 +4,9 @@ const duplexer2 = require('duplexer2'); const BufferStream = require('./BufferStream'); function parseOne(match, opts) { - const inStream = Stream.PassThrough({objectMode:true}); - const outStream = Stream.PassThrough(); - const transform = Stream.Transform({objectMode:true}); + const inStream = new Stream.PassThrough({objectMode:true}); + const outStream = new Stream.PassThrough(); + const transform = new Stream.Transform({objectMode:true}); const re = match instanceof RegExp ? match : (match && new RegExp(match)); let found; diff --git a/package.json b/package.json index 9f20cf8..82b944b 100644 --- a/package.json +++ b/package.json @@ -30,9 +30,10 @@ "graceful-fs": "^4.2.2" }, "devDependencies": { + "typescript": "^5.4.5", "@eslint/js": "^9.2.0", - "@types/node": "^20.12.11", "@types/bluebird": "^3.5.42", + "@types/node": "^20.12.11", "aws-sdk": "^2.77.0", "dirdiff": ">= 0.0.1 < 1", "eslint": "^9.2.0", diff --git a/test/compressed-crx.js b/test/compressed-crx.js index 865567e..2102717 100644 --- a/test/compressed-crx.js +++ b/test/compressed-crx.js @@ -42,17 +42,19 @@ test('open methods', async function(t) { const s3 = new AWS.S3({region: 'us-east-1'}); // We have to modify the `getObject` and `headObject` to use makeUnauthenticated + //@ts-ignore s3.getObject = function(params, cb) { return s3.makeUnauthenticatedRequest('getObject', params, cb); }; + //@ts-ignore s3.headObject = function(params, cb) { return s3.makeUnauthenticatedRequest('headObject', params, cb); }; const tests = [ - {name: 'buffer', args: [buffer]}, - {name: 'file', args: [archive]}, + {name: 'buffer', args: [buffer, {crx: true}]}, + {name: 'file', args: [archive, {crx: true}]}, // {name: 'url', args: [request, 'https://s3.amazonaws.com/unzipper/archive.crx']}, // {name: 's3', args: [s3, { Bucket: 'unzipper', Key: 'archive.crx'}]} ]; @@ -61,7 +63,7 @@ test('open methods', async function(t) { t.test(test.name, async function(t) { t.test('opening with crx option', function(t) { const method = unzip.Open[test.name]; - method.apply(method, test.args.concat({crx:true})) + method.apply(method, test.args) .then(function(d) { return d.files[1].buffer(); }) diff --git a/test/office-files.js b/test/office-files.js index b0a9fe4..719791e 100644 --- a/test/office-files.js +++ b/test/office-files.js @@ -19,14 +19,14 @@ test("get content a xlsx file without errors", async function () { test("stream retries when the local file header indicates bigger size than central directory", async function (t) { const archive = path.join(__dirname, '../testData/office/testfile.xlsx'); - let retries = 0, size; + let retries = 0, size = 0; const directory = await unzip.Open.file(archive, {padding: 10}); const stream = directory.files[0].stream(); stream.on('streamRetry', _size => { retries += 1; size = _size; }); - await new Promise(resolve => stream.pipe(NoopStream()).on('finish', resolve)); + await new Promise(resolve => stream.pipe(new NoopStream()).on('finish', resolve)); t.ok(retries === 1, 'retries once'); t.ok(size > 0, 'size is set'); }); \ No newline at end of file diff --git a/test/streamSingleEntry.js b/test/streamSingleEntry.js index 9fc4466..6a319c3 100644 --- a/test/streamSingleEntry.js +++ b/test/streamSingleEntry.js @@ -6,7 +6,7 @@ const unzip = require('../'); const Stream = require('stream'); test("pipe a single file entry out of a zip", function (t) { - const receiver = Stream.Transform({objectMode:true}); + const receiver = new Stream.Transform({objectMode:true}); receiver._transform = function(entry, e, cb) { if (entry.path === 'file.txt') { const writableStream = new streamBuffers.WritableStreamBuffer(); diff --git a/test/uncompressed.js b/test/uncompressed.js index 94af3fd..da113cc 100644 --- a/test/uncompressed.js +++ b/test/uncompressed.js @@ -63,7 +63,7 @@ test("do not extract zip slip archive", function (t) { fs.createReadStream(archive).pipe(unzipExtractor); function testNoSlip() { - const mode = fs.F_OK | (fs.constants && fs.constants.F_OK); + const mode = fs.constants.F_OK | (fs.constants && fs.constants.F_OK); return fs.access(path.join(os.tmpdir(), 'evil.txt'), mode, evilFileCallback); } @@ -105,7 +105,7 @@ function testZipSlipArchive(t, slipFileName, attackPathFactory){ function CheckForSlip(path, resultCallback) { const fsCallback = function(err){ return resultCallback(!err); }; - const mode = fs.F_OK | (fs.constants && fs.constants.F_OK); + const mode = fs.constants.F_OK | (fs.constants && fs.constants.F_OK); return fs.access(path, mode, fsCallback); }