diff --git a/.babelrc b/.babelrc new file mode 100644 index 00000000..5ab5512e --- /dev/null +++ b/.babelrc @@ -0,0 +1,9 @@ +{ + "env": { + "commonjs": { + "presets": [ + "@babel/env" + ] + } + } +} diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..4eb2283c --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +text eol=lf \ No newline at end of file diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml new file mode 100644 index 00000000..87566d8b --- /dev/null +++ b/.github/workflows/coverage.yml @@ -0,0 +1,50 @@ +name: Deploy coverate report to Pages + +on: + # Runs on pushes targeting the default branch + push: + branches: ["master"] + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages +permissions: + contents: read + pages: write + id-token: write + +# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued. +# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete. +concurrency: + group: "pages" + cancel-in-progress: false + +jobs: + # Single deploy job since we're just deploying + coverage: + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Use Node.js 18 + uses: actions/setup-node@v3 + with: + node-version: '18.x' + - run: npm install + - run: npm run build --if-present + - run: npm test + - run: npx lcov-badge2 coverage/lcov.info -o coverage/lcov-report/badge.svg + - name: Setup Pages + uses: actions/configure-pages@v3 + - name: Upload artifact + uses: actions/upload-pages-artifact@v2 + with: + path: 'coverage/lcov-report' + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v2 + \ No newline at end of file diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 00000000..294b573a --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,53 @@ +name: CI & Publish to NPM + +on: + push: + branches: [master] + workflow_dispatch: + +permissions: + contents: read + +jobs: + test: + uses: ./.github/workflows/test.yml + + publish: + runs-on: ubuntu-latest + needs: test + if: github.event_name == 'push' && github.ref == 'refs/heads/master' + permissions: + contents: read + id-token: write + + steps: + - uses: actions/checkout@v5 + + - uses: actions/setup-node@v5 + with: + node-version: 20.x + registry-url: https://registry.npmjs.org + package-manager-cache: false + + - name: Upgrade npm + run: | + npm install -g npm@latest + npm --version + node --version + + - name: Install dependencies + run: npm install + + - name: Publish package to npm + run: | + set -o pipefail + npm publish --provenance --access public 2>&1 | tee /tmp/npm-publish.log || EXIT=$? + + if [ "${EXIT:-0}" -ne 0 ] && grep -qiE "previously published|cannot publish over the previously published versions" /tmp/npm-publish.log; then + echo "Version already published, nothing to do" + exit 0 + fi + + exit "${EXIT:-0}" + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..34e5aaba --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,46 @@ +name: Node.js CI + +on: + pull_request: + types: [opened, synchronize, reopened] + workflow_call: + workflow_dispatch: + +permissions: + contents: read + +jobs: + lint: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v5 + + - uses: actions/setup-node@v5 + with: + node-version: 20.x + package-manager-cache: false + + - run: npm install + - run: npx eslint . + + test: + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + node-version: [10.x, 12.x, 14.x, 16.x, 17.x, 18.x, 19.x] + + steps: + - uses: actions/checkout@v5 + + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v5 + with: + node-version: ${{ matrix.node-version }} + package-manager-cache: false + + - run: npm install + - run: npm run build --if-present + - run: npm test diff --git a/.gitignore b/.gitignore index e0b850e0..02ad784f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,8 @@ /.idea -/node_modules \ No newline at end of file +/node_modules +/test.js +/.nyc_output/ +/coverage/ +/dist/ +.tap/ +package-lock.json diff --git a/.npmignore b/.npmignore new file mode 100644 index 00000000..ff2264e2 --- /dev/null +++ b/.npmignore @@ -0,0 +1,7 @@ +test/ +test-commonjs/ +testData/ +.nyc_output/ +coverage/ +.tap/ +.github/ \ No newline at end of file diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 5f6edf04..00000000 --- a/.travis.yml +++ /dev/null @@ -1,6 +0,0 @@ -language: node_js -node_js: - - "0.11" - - "0.10" - - "0.8" - diff --git a/LICENSE b/LICENSE index ccfbbcdc..20c59017 100644 --- a/LICENSE +++ b/LICENSE @@ -18,3 +18,8 @@ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +--- + +Commits in this fork are (c) Ziggy Jonsson (ziggy.jonsson.nyc@gmail.com) +and fall under same licence structure as the original repo (MIT) \ No newline at end of file diff --git a/README.md b/README.md index 9a3f7f88..30162831 100644 --- a/README.md +++ b/README.md @@ -1,40 +1,263 @@ -# unzip [![Build Status](https://travis-ci.org/EvanOxfeld/node-unzip.png)](https://travis-ci.org/EvanOxfeld/node-unzip) +[![NPM Version][npm-image]][npm-url] +[![NPM Downloads][downloads-image]][downloads-url] +[![code coverage](https://zjonsson.github.io/node-unzipper/badge.svg)](https://zjonsson.github.io/node-unzipper/) -Streaming cross-platform unzip tool written in node.js. +[npm-image]: https://img.shields.io/npm/v/unzipper.svg +[npm-url]: https://npmjs.org/package/unzipper +[travis-image]: https://api.travis-ci.org/ZJONSSON/node-unzipper.png?branch=master +[travis-url]: https://travis-ci.org/ZJONSSON/node-unzipper?branch=master +[downloads-image]: https://img.shields.io/npm/dm/unzipper.svg +[downloads-url]: https://npmjs.org/package/unzipper +[coverage-image]: https://3tjjj5abqi.execute-api.us-east-1.amazonaws.com/prod/node-unzipper/badge +[coverage-url]: https://3tjjj5abqi.execute-api.us-east-1.amazonaws.com/prod/node-unzipper/url -Unzip provides simple APIs similar to [node-tar](https://github.com/isaacs/node-tar) for parsing and extracting zip files. -There are no added compiled dependencies - inflation is handled by node.js's built in zlib support. Unzip is also an -example use case of [node-pullstream](https://github.com/EvanOxfeld/node-pullstream). +# unzipper ## Installation ```bash -$ npm install unzip +$ npm install unzipper ``` -## Quick Examples +## Open methods + +The open methods allow random access to the underlying files of a zip archive, from disk or from the web, s3 or a custom source. + +The open methods return a promise on the contents of the central directory of a zip file, with individual `files` listed in an array. + +Each file record has the following methods, providing random access to the underlying files: +* `stream([password])` - returns a stream of the unzipped content which can be piped to any destination +* `buffer([password])` - returns a promise on the buffered content of the file. + +If the file is encrypted you will have to supply a password to decrypt, otherwise you can leave blank. + +Unlike `adm-zip` the Open methods will never read the entire zipfile into buffer. + +The last argument to the `Open` methods is an optional `options` object where you can specify `tailSize` (default 80 bytes), i.e. how many bytes should we read at the end of the zipfile to locate the endOfCentralDirectory. This location can be variable depending on zip64 extensible data sector size. Additionally you can supply option `crx: true` which will check for a crx header and parse the file accordingly by shifting all file offsets by the length of the crx header. + + +### Open.file([path], [options]) + +Returns a Promise to the central directory information with methods to extract individual files. `start` and `end` options are used to avoid reading the whole file. + +Here is a simple example of opening up a zip file, printing out the directory information and then extracting the first file inside the zipfile to disk: +```js +async function main() { + const directory = await unzipper.Open.file('path/to/archive.zip'); + console.log('directory', directory); + return new Promise( (resolve, reject) => { + directory.files[0] + .stream() + .pipe(fs.createWriteStream('firstFile')) + .on('error',reject) + .on('close',resolve) + }); +} + +main(); +``` + +If you want to extract all files from the zip file, the directory object supplies an extract method. Here is a quick example: + +```js +async function main() { + const directory = await unzipper.Open.file('path/to/archive.zip'); + await directory.extract({ path: '/path/to/destination' }) +} +``` + + +### Open.url([requestLibrary], [url | params], [options]) + +This function will return a Promise to the central directory information from a URL point to a zipfile. Range-headers are used to avoid reading the whole file. Unzipper does not ship with a request library so you will have to provide it as the first option. + +Live Example: (extracts a tiny xml file from the middle of a 500MB zipfile) + +```js +const request = require('request'); +const unzipper = require('./unzip'); + +async function main() { + const directory = await unzipper.Open.url(request,'http://www2.census.gov/geo/tiger/TIGER2015/ZCTA5/tl_2015_us_zcta510.zip'); + const file = directory.files.find(d => d.path === 'tl_2015_us_zcta510.shp.iso.xml'); + const content = await file.buffer(); + console.log(content.toString()); +} + +main(); +``` + + +This function takes a second parameter which can either be a string containing the `url` to request, or an `options` object to invoke the supplied `request` library with. This can be used when other request options are required, such as custom headers or authentication to a third party service. + +```js +const request = require('google-oauth-jwt').requestWithJWT(); + +const googleStorageOptions = { + url: `https://www.googleapis.com/storage/v1/b/m-bucket-name/o/my-object-name`, + qs: { alt: 'media' }, + jwt: { + email: google.storage.credentials.client_email, + key: google.storage.credentials.private_key, + scopes: ['https://www.googleapis.com/auth/devstorage.read_only'] + } +}; + +async function getFile(req, res, next) { + const directory = await unzipper.Open.url(request, googleStorageOptions); + const file = zip.files.find((file) => file.path === 'my-filename'); + return file.stream().pipe(res); +}; +``` + + +### Open.s3([aws-sdk], [params], [options]) + +This function will return a Promise to the central directory information from a zipfile on S3. Range-headers are used to avoid reading the whole file. Unzipper does not ship with with the aws-sdk so you have to provide an instantiated client as first arguments. The params object requires `Bucket` and `Key` to fetch the correct file. + +Example: + +```js +const unzipper = require('./unzip'); +const AWS = require('aws-sdk'); +const s3Client = AWS.S3(config); + +async function main() { + const directory = await unzipper.Open.s3(s3Client,{Bucket: 'unzipper', Key: 'archive.zip'}); + return new Promise( (resolve, reject) => { + directory.files[0] + .stream() + .pipe(fs.createWriteStream('firstFile')) + .on('error',reject) + .on('close',resolve) + }); +} + +main(); +``` + + +### Open.buffer(buffer, [options]) + +If you already have the zip file in-memory as a buffer, you can open the contents directly. + +Example: + +```js +// never use readFileSync - only used here to simplify the example +const buffer = fs.readFileSync('path/to/arhive.zip'); + +async function main() { + const directory = await unzipper.Open.buffer(buffer); + console.log('directory',directory); + // ... +} + +main(); +``` + + +### Open.custom(source, [options]) + +This function can be used to provide a custom source implementation. The source parameter expects a `stream` and a `size` function to be implemented. The size function should return a `Promise` that resolves the total size of the file. The stream function should return a `Readable` stream according to the supplied offset and length parameters. + +Example: + +```js +// Custom source implementation for reading a zip file from Google Cloud Storage +const { Storage } = require('@google-cloud/storage'); + +async function main() { + const storage = new Storage(); + const bucket = storage.bucket('my-bucket'); + const zipFile = bucket.file('my-zip-file.zip'); + + const customSource = { + stream: function(offset, length) { + return zipFile.createReadStream({ + start: offset, + end: length && offset + length + }) + }, + size: async function() { + const objMetadata = (await zipFile.getMetadata())[0]; + return objMetadata.size; + } + }; + + const directory = await unzipper.Open.custom(customSource); + console.log('directory', directory); + // ... +} + +main(); +``` + + +### Open.[method].extract() + +The directory object returned from `Open.[method]` provides an `extract` method which extracts all the files to a specified `path`, with an optional `concurrency` (default: 1). + +Example (with concurrency of 5): + +```js +unzip.Open.file('path/to/archive.zip') + .then(directory => directory.extract({path: '/extraction/path', concurrency: 5})); +``` + + +Please note: Methods that use the Central Directory instead of parsing entire file can be found under [`Open`](#open) + +Chrome extension files (.crx) are zipfiles with an [extra header](http://www.adambarth.com/experimental/crx/docs/crx.html) at the start of the file. Unzipper will parse .crx file with the streaming methods (`Parse` and `ParseOne`). + + +## Streaming an entire zip file (legacy) + +This library began as an active fork and drop-in replacement of the [node-unzip](https://github.com/EvanOxfeld/node-unzip) to address the following issues: +* finish/close events are not always triggered, particular when the input stream is slower than the receivers +* Any files are buffered into memory before passing on to entry + +Originally the only way to use the library was to stream the entire zip file. This method is inefficient if you are only interested in selected files from the zip files. Additionally this method can be error prone since it relies on the local file headers which could be wrong. + +The structure of this fork is similar to the original, but uses Promises and inherit guarantees provided by node streams to ensure low memory footprint and emits finish/close events at the end of processing. The new `Parser` will push any parsed `entries` downstream if you pipe from it, while still supporting the legacy `entry` event as well. + +Breaking changes: The new `Parser` will not automatically drain entries if there are no listeners or pipes in place. + +Unzipper provides simple APIs similar to [node-tar](https://github.com/isaacs/node-tar) for parsing and extracting zip files. +There are no added compiled dependencies - inflation is handled by node.js's built in zlib support. ### Extract to a directory -```javascript -fs.createReadStream('path/to/archive.zip').pipe(unzip.Extract({ path: 'output/path' })); +```js +fs.createReadStream('path/to/archive.zip') + .pipe(unzipper.Extract({ path: 'output/path' })); ``` -Extract emits the 'close' event once the zip's contents have been fully extracted to disk. +Extract emits the 'close' event once the zip's contents have been fully extracted to disk. `Extract` uses [fstream.Writer](https://www.npmjs.com/package/fstream) and therefore needs an absolute path to the destination directory. This directory will be automatically created if it doesn't already exist. ### Parse zip file contents Process each zip file entry or pipe entries to another stream. __Important__: If you do not intend to consume an entry stream's raw data, call autodrain() to dispose of the entry's -contents. Otherwise you risk running out of memory. +contents. Otherwise the stream will halt. `.autodrain()` returns an empty stream that provides `error` and `finish` events. +Additionally you can call `.autodrain().promise()` to get the promisified version of success or failure of the autodrain. -```javascript +```js +// If you want to handle autodrain errors you can either: +entry.autodrain().catch(e => handleError); +// or +entry.autodrain().on('error' => handleError); +``` + +Here is a quick example: + +```js fs.createReadStream('path/to/archive.zip') - .pipe(unzip.Parse()) + .pipe(unzipper.Parse()) .on('entry', function (entry) { - var fileName = entry.path; - var type = entry.type; // 'Directory' or 'File' - var size = entry.size; + const fileName = entry.path; + const type = entry.type; // 'Directory' or 'File' + const size = entry.vars.uncompressedSize; // There is also compressedSize; if (fileName === "this IS the file I'm looking for") { entry.pipe(fs.createWriteStream('output/path')); } else { @@ -43,39 +266,131 @@ fs.createReadStream('path/to/archive.zip') }); ``` -Or pipe the output of unzip.Parse() to fstream +and the same example using async iterators: + +```js +const zip = fs.createReadStream('path/to/archive.zip').pipe(unzipper.Parse({forceStream: true})); +for await (const entry of zip) { + const fileName = entry.path; + const type = entry.type; // 'Directory' or 'File' + const size = entry.vars.uncompressedSize; // There is also compressedSize; + if (fileName === "this IS the file I'm looking for") { + entry.pipe(fs.createWriteStream('output/path')); + } else { + entry.autodrain(); + } +} +``` + +### Parse zip by piping entries downstream -```javascript -var readStream = fs.createReadStream('path/to/archive.zip'); -var writeStream = fstream.Writer('output/path'); +If you `pipe` from unzipper the downstream components will receive each `entry` for further processing. This allows for clean pipelines transforming zipfiles into unzipped data. -readStream - .pipe(unzip.Parse()) - .pipe(writeStream) +Example using `stream.Transform`: + +```js +fs.createReadStream('path/to/archive.zip') + .pipe(unzipper.Parse()) + .pipe(stream.Transform({ + objectMode: true, + transform: function(entry,e,cb) { + const fileName = entry.path; + const type = entry.type; // 'Directory' or 'File' + const size = entry.vars.uncompressedSize; // There is also compressedSize; + if (fileName === "this IS the file I'm looking for") { + entry.pipe(fs.createWriteStream('output/path')) + .on('close',cb); + } else { + entry.autodrain(); + cb(); + } + } + })); ``` -## License +Example using [etl](https://www.npmjs.com/package/etl): -(The MIT License) +```js +fs.createReadStream('path/to/archive.zip') + .pipe(unzipper.Parse()) + .pipe(etl.map(entry => { + if (entry.path == "this IS the file I'm looking for") + return entry + .pipe(etl.toFile('output/path')) + .promise(); + else + entry.autodrain(); + })) -Copyright (c) 2012 - 2013 Near Infinity Corporation +``` + +### Parse a single file and pipe contents + +`unzipper.parseOne([regex])` is a convenience method that unzips only one file from the archive and pipes the contents down (not the entry itself). If no search criteria is specified, the first file in the archive will be unzipped. Otherwise, each filename will be compared to the criteria and the first one to match will be unzipped and piped down. If no file matches then the the stream will end without any content. + +Example: + +```js +fs.createReadStream('path/to/archive.zip') + .pipe(unzipper.ParseOne()) + .pipe(fs.createWriteStream('firstFile.txt')); +``` + +### Buffering the content of an entry into memory + +While the recommended strategy of consuming the unzipped contents is using streams, it is sometimes convenient to be able to get the full buffered contents of each file . Each `entry` provides a `.buffer` function that consumes the entry by buffering the contents into memory and returning a promise to the complete buffer. + +```js +fs.createReadStream('path/to/archive.zip') + .pipe(unzipper.Parse()) + .pipe(etl.map(async entry => { + if (entry.path == "this IS the file I'm looking for") { + const content = await entry.buffer(); + await fs.writeFile('output/path',content); + } + else { + entry.autodrain(); + } + })) +``` + +### Parse.promise() syntax sugar + +The parser emits `finish` and `error` events like any other stream. The parser additionally provides a promise wrapper around those two events to allow easy folding into existing Promise-based structures. + +Example: + +```js +fs.createReadStream('path/to/archive.zip') + .pipe(unzipper.Parse()) + .on('entry', entry => entry.autodrain()) + .promise() + .then( () => console.log('done'), e => console.log('error',e)); +``` -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: +### Parse zip created by DOS ZIP or Windows ZIP Folders -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. +Archives created by legacy tools usually have filenames encoded with IBM PC (Windows OEM) character set. +You can decode filenames with preferred character set: -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +```js +const il = require('iconv-lite'); +fs.createReadStream('path/to/archive.zip') + .pipe(unzipper.Parse()) + .on('entry', function (entry) { + // if some legacy zip tool follow ZIP spec then this flag will be set + const isUnicode = entry.props.flags.isUnicode; + // decode "non-unicode" filename from OEM Cyrillic character set + const fileName = isUnicode ? entry.path : il.decode(entry.props.pathBuffer, 'cp866'); + const type = entry.type; // 'Directory' or 'File' + const size = entry.vars.uncompressedSize; // There is also compressedSize; + if (fileName === "Текстовый файл.txt") { + entry.pipe(fs.createWriteStream(fileName)); + } else { + entry.autodrain(); + } + }); +``` +## Licenses +See LICENCE diff --git a/eslint.config.mjs b/eslint.config.mjs new file mode 100644 index 00000000..1b4f5ad4 --- /dev/null +++ b/eslint.config.mjs @@ -0,0 +1,29 @@ +import globals from "globals"; +import pluginJs from "@eslint/js"; + + +export default [ + { + rules:{ + "semi": 2, + "no-multiple-empty-lines": 2, + "no-multi-spaces": 2, + "comma-spacing": 2, + "prefer-const": 2, + "no-trailing-spaces": 2, + "no-var": 2, + "indent": [ + "error", + 2, + { + "MemberExpression": 1, + "SwitchCase": 1, + "ignoredNodes": ["TemplateLiteral > *"] + } + ], + } + }, + {files: ["**/*.js"], languageOptions: {sourceType: "module"}}, + {languageOptions: { globals: globals.node }}, + pluginJs.configs.recommended, +]; diff --git a/index.cjs b/index.cjs new file mode 100644 index 00000000..63985a9b --- /dev/null +++ b/index.cjs @@ -0,0 +1,5 @@ +'use strict'; +exports.Parse = require('./dist/parse.js').default; +exports.ParseOne = require('./dist/parseOne.js').default; +exports.Extract = require('./dist/extract.js').default; +exports.Open = require('./dist/Open/index.js').default; diff --git a/index.js b/index.js new file mode 100644 index 00000000..fad8d849 --- /dev/null +++ b/index.js @@ -0,0 +1,4 @@ +export { default as Parse } from './lib/parse.js'; +export { default as ParseOne } from './lib/parseOne.js'; +export { default as Extract } from './lib/extract.js'; +export { default as Open } from './lib/Open/index.js'; diff --git a/lib/BufferStream.js b/lib/BufferStream.js new file mode 100644 index 00000000..b8831d45 --- /dev/null +++ b/lib/BufferStream.js @@ -0,0 +1,28 @@ +import { Transform } from 'stream'; // 'node:stream' + +/** + * Reads an entry as a `Buffer`. + * @param {*} entry + * @returns {Promise} + */ +export default function readAsBuffer(entry) { + return new Promise(function(resolve, reject) { + const chunks = []; + + const bufferStream = new Transform({ + transform + }) + .on('finish', function() { + resolve(Buffer.concat(chunks)); + }) + .on('error', reject); + + function transform(d, e, cb) { + chunks.push(d); + cb(); + }; + + entry.on('error', reject) + .pipe(bufferStream); + }); +}; diff --git a/lib/Decrypt.js b/lib/Decrypt.js new file mode 100644 index 00000000..e0fba90c --- /dev/null +++ b/lib/Decrypt.js @@ -0,0 +1,103 @@ +import Int64 from "node-int64"; +import { Transform } from "stream"; // "node:stream" + +let table; + +function generateTable() { + const poly = 0xEDB88320; + let c, n, k; + table = []; + for (n = 0; n < 256; n++) { + c = n; + for (k = 0; k < 8; k++) c = c & 1 ? poly ^ (c >>> 1) : (c = c >>> 1); + table[n] = c >>> 0; + } +} + +function crc(ch, crc) { + if (!table) generateTable(); + + if (ch.charCodeAt) ch = ch.charCodeAt(0); + + const l = (crc.readUInt32BE() >> 8) & 0xffffff; + const r = table[(crc.readUInt32BE() ^ (ch >>> 0)) & 0xff]; + + return (l ^ r) >>> 0; +} + +function multiply(a, b) { + const ah = (a >> 16) & 0xffff; + const al = a & 0xffff; + const bh = (b >> 16) & 0xffff; + const bl = b & 0xffff; + const high = (ah * bl + al * bh) & 0xffff; + + return ((high << 16) >>> 0) + al * bl; +} + +const DecryptConstructor = function () { + this.key0 = Buffer.allocUnsafe(4); + this.key1 = Buffer.allocUnsafe(4); + this.key2 = Buffer.allocUnsafe(4); + + this.key0.writeUInt32BE(0x12345678, 0); + this.key1.writeUInt32BE(0x23456789, 0); + this.key2.writeUInt32BE(0x34567890, 0); +}; + +const update = function (h) { + this.key0.writeUInt32BE(crc(h, this.key0)); + this.key1.writeUInt32BE( + ((this.key0.readUInt32BE() & 0xff & 0xFFFFFFFF) + + this.key1.readUInt32BE()) >>> 0 + ); + const x = new Int64( + (multiply(this.key1.readUInt32BE(), 134775813) + 1) & 0xFFFFFFFF + ); + const b = Buffer.alloc(8); + x.copy(b, 0); + b.copy(this.key1, 0, 4, 8); + this.key2.writeUInt32BE( + crc(((this.key1.readUInt32BE() >> 24) & 0xff) >>> 0, this.key2) + ); +}; + +const decryptByte = function (c) { + const k = (this.key2.readUInt32BE() | 2) >>> 0; + c = c ^ ((multiply(k, (k ^ 1 >>> 0)) >> 8) & 0xff); + this.update(c); + + return c; +}; + +const stream = function () { + const self = this; + const transform = function (d, e, cb) { + for (let i = 0; i < d.length; i++) { + d[i] = self.decryptByte(d[i]); + } + this.push(d); + cb(); + }; + return new Transform({ + transform + }); +}; + +export default class Decrypt { + constructor() { + DecryptConstructor.call(this); + } + + update(h) { + return update.call(this, h); + }; + + decryptByte(c) { + return decryptByte.call(this, c); + }; + + stream() { + return stream.call(this); + }; +} diff --git a/lib/NoopStream.js b/lib/NoopStream.js new file mode 100644 index 00000000..fbb13f17 --- /dev/null +++ b/lib/NoopStream.js @@ -0,0 +1,5 @@ +import { Transform } from 'stream'; // 'node:stream' + +export default class NoopStream extends Transform { + _transform(d, e, cb) { cb() ;}; +} \ No newline at end of file diff --git a/lib/Open/directory.js b/lib/Open/directory.js new file mode 100644 index 00000000..4c35e850 --- /dev/null +++ b/lib/Open/directory.js @@ -0,0 +1,251 @@ + +import path from 'path'; // 'node:path' +import fs from 'fs'; // 'node:fs' +import fsExtra from 'fs-extra'; +import Bluebird from 'bluebird'; + +import unzip from './unzip.js'; +import PullStream from '../PullStream.js'; +import BufferStream from '../BufferStream.js'; +import parseExtraField from '../parseExtraField.js'; +import parseDateTime from '../parseDateTime.js'; +import parseBuffer from '../parseBuffer.js'; + +const signature = Buffer.alloc(4); +signature.writeUInt32LE(0x06054b50, 0); + +function getCrxHeader(source) { + const sourceStream = source.stream(0).pipe(new PullStream()); + + return sourceStream.pull(4).then(function(data) { + const signature = data.readUInt32LE(0); + if (signature === 0x34327243) { + let crxHeader; + return sourceStream.pull(12).then(function(data) { + crxHeader = parseBuffer(data, [ + ['version', 4], + ['pubKeyLength', 4], + ['signatureLength', 4], + ]); + }).then(function() { + return sourceStream.pull(crxHeader.pubKeyLength +crxHeader.signatureLength); + }).then(function(data) { + crxHeader.publicKey = data.slice(0, crxHeader.pubKeyLength); + crxHeader.signature = data.slice(crxHeader.pubKeyLength); + crxHeader.size = 16 + crxHeader.pubKeyLength +crxHeader.signatureLength; + return crxHeader; + }); + } + }); +} + +// Zip64 File Format Notes: https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT +function getZip64CentralDirectory(source, zip64CDL) { + const d64loc = parseBuffer(zip64CDL, [ + ['signature', 4], + ['diskNumber', 4], + ['offsetToStartOfCentralDirectory', 8], + ['numberOfDisks', 4], + ]); + + if (d64loc.signature != 0x07064b50) { + throw new Error('invalid zip64 end of central dir locator signature (0x07064b50): 0x' + d64loc.signature.toString(16)); + } + + const dir64 = new PullStream(); + source.stream(d64loc.offsetToStartOfCentralDirectory).pipe(dir64); + + return dir64.pull(56); +} + +// Zip64 File Format Notes: https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT +function parseZip64DirRecord (dir64record) { + const vars = parseBuffer(dir64record, [ + ['signature', 4], + ['sizeOfCentralDirectory', 8], + ['version', 2], + ['versionsNeededToExtract', 2], + ['diskNumber', 4], + ['diskStart', 4], + ['numberOfRecordsOnDisk', 8], + ['numberOfRecords', 8], + ['sizeOfCentralDirectory', 8], + ['offsetToStartOfCentralDirectory', 8], + ]); + + if (vars.signature != 0x06064b50) { + throw new Error('invalid zip64 end of central dir locator signature (0x06064b50): 0x0' + vars.signature.toString(16)); + } + + return vars; +} + +export default function centralDirectory(source, options) { + const endDir = new PullStream(); + const records = new PullStream(); + const tailSize = (options && options.tailSize) || 80; + let sourceSize, + crxHeader, + startOffset, + vars; + + if (options && options.crx) + crxHeader = getCrxHeader(source); + + return source.size() + .then(function(size) { + sourceSize = size; + + source.stream(Math.max(0, size-tailSize)) + .on('error', function (error) { endDir.emit('error', error); }) + .pipe(endDir); + + return endDir.pull(signature); + }) + .then(function() { + return Promise.all([endDir.pull(22), crxHeader]); + }) + .then(function([directory, crxHeader]) { + const data = directory; + startOffset = crxHeader && crxHeader.size || 0; + + vars = parseBuffer(data, [ + ['signature', 4], + ['diskNumber', 2], + ['diskStart', 2], + ['numberOfRecordsOnDisk', 2], + ['numberOfRecords', 2], + ['sizeOfCentralDirectory', 4], + ['offsetToStartOfCentralDirectory', 4], + ['commentLength', 2], + ]); + + // Is this zip file using zip64 format? Use same check as Go: + // https://github.com/golang/go/blob/master/src/archive/zip/reader.go#L503 + // For zip64 files, need to find zip64 central directory locator header to extract + // relative offset for zip64 central directory record. + if (vars.diskNumber == 0xffff || vars.numberOfRecords == 0xffff || + vars.offsetToStartOfCentralDirectory == 0xffffffff) { + + // Offset to zip64 CDL is 20 bytes before normal CDR + const zip64CDLSize = 20; + const zip64CDLOffset = sourceSize - (tailSize - endDir.match + zip64CDLSize); + const zip64CDLStream = new PullStream(); + + source.stream(zip64CDLOffset).pipe(zip64CDLStream); + + return zip64CDLStream.pull(zip64CDLSize) + .then(function (d) { return getZip64CentralDirectory(source, d); }) + .then(function (dir64record) { + vars = parseZip64DirRecord(dir64record); + }); + } else { + vars.offsetToStartOfCentralDirectory += startOffset; + } + }) + .then(function() { + if (vars.commentLength) return endDir.pull(vars.commentLength).then(function(comment) { + vars.comment = comment.toString('utf8'); + }); + }) + .then(async function() { + source.stream(vars.offsetToStartOfCentralDirectory).pipe(records); + + vars.extract = async function(opts) { + if (!opts || !opts.path) throw new Error('PATH_MISSING'); + // make sure path is normalized before using it + opts.path = path.resolve(path.normalize(opts.path)); + const files = vars.files; + return Promise.resolve( + Bluebird.map(files, async function(entry) { + // to avoid zip slip (writing outside of the destination), we resolve + // the target path, and make sure it's nested in the intended + // destination, or not extract it otherwise. + const extractPath = path.join(opts.path, entry.path.replace(/\\/g, '/')); + const rel = path.relative(opts.path, extractPath); + if (rel === '' || rel.startsWith('..') || path.isAbsolute(rel)) { + return; + } + + if (entry.type == 'Directory') { + await fsExtra.ensureDir(extractPath); + return; + } + + await fsExtra.ensureDir(path.dirname(extractPath)); + + const writer = opts.getWriter ? opts.getWriter({path: extractPath}) : fs.createWriteStream(extractPath); + + return new Promise(function(resolve, reject) { + entry.stream(opts.password) + .on('error', reject) + .pipe(writer) + .on('close', resolve) + .on('error', reject); + }); + }, { concurrency: opts.concurrency > 1 ? opts.concurrency : 1 }) + ); + }; + + vars.files = []; + let i = 0; + while (i < vars.numberOfRecords) { + vars.files[i] = await records.pull(46).then(function(data) { + const vars = parseBuffer(data, [ + ['signature', 4], + ['versionMadeBy', 2], + ['versionsNeededToExtract', 2], + ['flags', 2], + ['compressionMethod', 2], + ['lastModifiedTime', 2], + ['lastModifiedDate', 2], + ['crc32', 4], + ['compressedSize', 4], + ['uncompressedSize', 4], + ['fileNameLength', 2], + ['extraFieldLength', 2], + ['fileCommentLength', 2], + ['diskNumber', 2], + ['internalFileAttributes', 2], + ['externalFileAttributes', 4], + ['offsetToLocalFileHeader', 4], + ]); + + vars.offsetToLocalFileHeader += startOffset; + vars.lastModifiedDateTime = parseDateTime(vars.lastModifiedDate, vars.lastModifiedTime); + + return records.pull(vars.fileNameLength).then(function(fileNameBuffer) { + vars.pathBuffer = fileNameBuffer; + vars.path = fileNameBuffer.toString('utf8'); + vars.isUnicode = (vars.flags & 0x800) != 0; + return records.pull(vars.extraFieldLength); + }) + .then(function(extraField) { + vars.extra = parseExtraField(extraField, vars); + return records.pull(vars.fileCommentLength); + }) + .then(function(comment) { + vars.comment = comment; + vars.type = (vars.uncompressedSize === 0 && /[/\\]$/.test(vars.path)) ? 'Directory' : 'File'; + const padding = options && options.padding || 1000; + vars.stream = function(_password) { + const totalSize = 30 + + padding // add an extra buffer + + (vars.extraFieldLength || 0) + + (vars.fileNameLength || 0) + + vars.compressedSize; + + return unzip(source, vars.offsetToLocalFileHeader, _password, vars, totalSize); + }; + vars.buffer = function(_password) { + return BufferStream(vars.stream(_password)); + }; + return vars; + }); + }); + i++; + } + + return vars; + }); +}; diff --git a/lib/Open/index.js b/lib/Open/index.js new file mode 100644 index 00000000..b6303a0b --- /dev/null +++ b/lib/Open/index.js @@ -0,0 +1,142 @@ +import fs from 'graceful-fs'; +import { PassThrough } from 'stream'; // 'node:stream' + +import directory from './directory.js'; + +export default { + buffer: function(buffer, options) { + const source = { + stream: function(offset, length) { + const stream = new PassThrough(); + const end = length ? offset + length : undefined; + stream.end(buffer.slice(offset, end)); + return stream; + }, + size: function() { + return Promise.resolve(buffer.length); + } + }; + return directory(source, options); + }, + file: function(filename, options) { + const source = { + stream: function(start, length) { + const end = length ? start + length : undefined; + return fs.createReadStream(filename, {start, end}); + }, + size: function() { + return new Promise(function(resolve, reject) { + fs.stat(filename, function(err, d) { + if (err) + reject(err); + else + resolve(d.size); + }); + }); + } + }; + return directory(source, options); + }, + + url: function(request, params, options) { + if (typeof params === 'string') + params = {url: params}; + if (!params.url) + throw 'URL missing'; + params.headers = params.headers || {}; + + const source = { + stream : function(offset, length) { + const options = Object.create(params); + const end = length ? offset + length : ''; + options.headers = Object.create(params.headers); + options.headers.range = 'bytes='+offset+'-' + end; + return request(options); + }, + size: function() { + return new Promise(function(resolve, reject) { + const req = request(params); + req.on('response', function(d) { + req.abort(); + if (!d.headers['content-length']) + reject(new Error('Missing content length header')); + else + resolve(d.headers['content-length']); + }).on('error', reject); + }); + } + }; + + return directory(source, options); + }, + + s3 : function(client, params, options) { + const source = { + size: function() { + return new Promise(function(resolve, reject) { + client.headObject(params, function(err, d) { + if (err) + reject(err); + else + resolve(d.ContentLength); + }); + }); + }, + stream: function(offset, length) { + const d = {}; + for (const key in params) + d[key] = params[key]; + const end = length ? offset + length : ''; + d.Range = 'bytes='+offset+'-' + end; + return client.getObject(d).createReadStream(); + } + }; + + return directory(source, options); + }, + s3_v3: async function (client, params, options) { + //@ts-ignore + const { GetObjectCommand, HeadObjectCommand } = await import('@aws-sdk/client-s3'); + const source = { + size: async () => { + const head = await client.send( + new HeadObjectCommand({ + Bucket: params.Bucket, + Key: params.Key, + }) + ); + + if(!head.ContentLength) { + return 0; + } + + return head.ContentLength; + }, + stream: (offset, length) => { + const stream = new PassThrough(); + const end = length ? offset + length : ""; + client + .send( + new GetObjectCommand({ + Bucket: params.Bucket, + Key: params.Key, + Range: `bytes=${offset}-${end}`, + }) + ) + .then((response) => { + response.Body.pipe(stream); + }) + .catch((error) => { + stream.emit("error", error); + }); + + return stream; + }, + }; + + return directory(source, options); + }, + custom: function(source, options) { + return directory(source, options); + } +}; diff --git a/lib/Open/unzip.js b/lib/Open/unzip.js new file mode 100644 index 00000000..73ba752b --- /dev/null +++ b/lib/Open/unzip.js @@ -0,0 +1,121 @@ +import { PassThrough } from 'stream'; // 'node:stream' +import zlib from 'zlib'; // 'node:zlib' + +import Decrypt from '../Decrypt.js'; +import PullStream from '../PullStream.js'; +import parseExtraField from '../parseExtraField.js'; +import parseDateTime from '../parseDateTime.js'; +import parseBuffer from '../parseBuffer.js'; + +export default function unzip(source, offset, _password, directoryVars, length) { + const file = new PullStream(), + entry = new PassThrough(); + + const req = source.stream(offset, length); + req.pipe(file).on('error', function(e) { + entry.emit('error', e); + }); + + entry.vars = file.pull(30) + .then(function(data) { + let vars = parseBuffer(data, [ + ['signature', 4], + ['versionsNeededToExtract', 2], + ['flags', 2], + ['compressionMethod', 2], + ['lastModifiedTime', 2], + ['lastModifiedDate', 2], + ['crc32', 4], + ['compressedSize', 4], + ['uncompressedSize', 4], + ['fileNameLength', 2], + ['extraFieldLength', 2], + ]); + + vars.lastModifiedDateTime = parseDateTime(vars.lastModifiedDate, vars.lastModifiedTime); + + return file.pull(vars.fileNameLength) + .then(function(fileName) { + vars.fileName = fileName.toString('utf8'); + return file.pull(vars.extraFieldLength); + }) + .then(function(extraField) { + let checkEncryption; + vars.extra = parseExtraField(extraField, vars); + // Ignore logal file header vars if the directory vars are available + if (directoryVars && directoryVars.compressedSize) vars = directoryVars; + + if (vars.flags & 0x01) checkEncryption = file.pull(12) + .then(function(header) { + if (!_password) + throw new Error('MISSING_PASSWORD'); + + const decrypt = new Decrypt(); + + String(_password).split('').forEach(function(d) { + decrypt.update(d); + }); + + for (let i=0; i < header.length; i++) + header[i] = decrypt.decryptByte(header[i]); + + vars.decrypt = decrypt; + vars.compressedSize -= 12; + + const check = (vars.flags & 0x8) ? (vars.lastModifiedTime >> 8) & 0xff : (vars.crc32 >> 24) & 0xff; + if (header[11] !== check) + throw new Error('BAD_PASSWORD'); + + return vars; + }); + + return Promise.resolve(checkEncryption) + .then(function() { + entry.emit('vars', vars); + return vars; + }); + }); + }); + + entry.vars.then(function(vars) { + const fileSizeKnown = !(vars.flags & 0x08) || vars.compressedSize > 0; + let eof; + + const inflater = vars.compressionMethod ? zlib.createInflateRaw() : new PassThrough(); + + if (fileSizeKnown) { + entry.size = vars.uncompressedSize; + eof = vars.compressedSize; + } else { + eof = Buffer.alloc(4); + eof.writeUInt32LE(0x08074b50, 0); + } + + let stream = file.stream(eof); + + if (vars.decrypt) + stream = stream.pipe(vars.decrypt.stream()); + + stream + .pipe(inflater) + .on('error', function(err) { entry.emit('error', err);}) + .pipe(entry) + .on('finish', function() { + if(req.destroy) + req.destroy(); + else if (req.abort) + req.abort(); + else if (req.close) + req.close(); + else if (req.push) + req.push(); + else + console.log('warning - unable to close stream'); + }); + }) + .catch(function(e) { + entry.emit('error', e); + }); + + return entry; +}; diff --git a/lib/PullStream.js b/lib/PullStream.js new file mode 100644 index 00000000..9edc8e46 --- /dev/null +++ b/lib/PullStream.js @@ -0,0 +1,156 @@ +import { Duplex, PassThrough, Transform } from 'stream'; // 'node:stream' + +// It's not clear why did they extract a string into a variable. +const FUNCTION_STRING = 'function'; + +const PullStreamConstructor = function() { + this.buffer = Buffer.from(''); + + const self = this; + self.on('finish', function() { + self.finished = true; + self.emit('chunk', false); + }); +}; + +const _write = function(chunk, e, cb) { + this.buffer = Buffer.concat([this.buffer, chunk]); + this.cb = cb; + this.emit('chunk'); +}; + +// The `eof` parameter is interpreted as `file_length` if the type is number +// otherwise (i.e. buffer) it is interpreted as a pattern signaling end of stream +const stream = function(eof, includeEof) { + const p = new PassThrough(); + let done; + const self = this; + + function cb() { + if (typeof self.cb === FUNCTION_STRING) { + const callback = self.cb; + self.cb = undefined; + return callback(); + } + } + + function pull() { + let packet; + if (self.buffer && self.buffer.length) { + if (typeof eof === 'number') { + packet = self.buffer.slice(0, eof); + self.buffer = self.buffer.slice(eof); + eof -= packet.length; + done = done || !eof; + } else { + let match = self.buffer.indexOf(eof); + if (match !== -1) { + // store signature match byte offset to allow us to reference + // this for zip64 offset + self.match = match; + if (includeEof) match = match + eof.length; + packet = self.buffer.slice(0, match); + self.buffer = self.buffer.slice(match); + done = true; + } else { + const len = self.buffer.length - eof.length; + if (len <= 0) { + cb(); + } else { + packet = self.buffer.slice(0, len); + self.buffer = self.buffer.slice(len); + } + } + } + if (packet) p.write(packet, function() { + if (self.buffer.length === 0 || (eof.length && self.buffer.length <= eof.length)) cb(); + }); + } + + if (!done) { + if (self.finished) { + self.removeListener('chunk', pull); + self.emit('error', new Error('FILE_ENDED')); + return; + } + + } else { + self.removeListener('chunk', pull); + p.end(); + } + } + + self.on('chunk', pull); + pull(); + return p; +}; + +const pull = function(eof, includeEof) { + if (eof === 0) return Promise.resolve(''); + + // If we already have the required data in buffer + // we can resolve the request immediately + if (!isNaN(eof) && this.buffer.length > eof) { + const data = this.buffer.slice(0, eof); + this.buffer = this.buffer.slice(eof); + return Promise.resolve(data); + } + + // Otherwise we stream until we have it + let buffer = Buffer.from(''); + const self = this; + + const concatStream = new Transform({ + transform + }); + + function transform (d, e, cb) { + buffer = Buffer.concat([buffer, d]); + cb(); + } + + let rejectHandler; + let pullStreamRejectHandler; + return new Promise(function(resolve, reject) { + rejectHandler = reject; + pullStreamRejectHandler = function(e) { + self.__emittedError = e; + reject(e); + }; + if (self.finished) + return reject(new Error('FILE_ENDED')); + self.once('error', pullStreamRejectHandler); // reject any errors from pullstream itself + self.stream(eof, includeEof) + .on('error', reject) + .pipe(concatStream) + .on('finish', function() {resolve(buffer);}) + .on('error', reject); + }) + .finally(function() { + self.removeListener('error', rejectHandler); + self.removeListener('error', pullStreamRejectHandler); + }); +}; + +export default class PullStream extends Duplex { + constructor() { + super({ decodeStrings: false, objectMode: true }); + PullStreamConstructor.call(this); + } + + _write(chunk, e, cb) { + return _write.call(this, chunk, e, cb); + }; + + // The `eof` parameter is interpreted as `file_length` if the type is number + // otherwise (i.e. buffer) it is interpreted as a pattern signaling end of stream + stream(eof, includeEof) { + return stream.call(this, eof, includeEof); + }; + + pull(eof, includeEof) { + return pull.call(this, eof, includeEof); + }; + + _read() {}; +} diff --git a/lib/entry.js b/lib/entry.js deleted file mode 100644 index 43535a81..00000000 --- a/lib/entry.js +++ /dev/null @@ -1,17 +0,0 @@ -'use strict'; - -module.exports = Entry; - -var PassThrough = require('readable-stream/passthrough'); -var inherits = require('util').inherits; - -inherits(Entry, PassThrough); - -function Entry () { - PassThrough.call(this); - this.props = {}; -} - -Entry.prototype.autodrain = function () { - this.on('readable', this.read.bind(this)); -}; diff --git a/lib/extract.js b/lib/extract.js index fd031813..1069552b 100644 --- a/lib/extract.js +++ b/lib/extract.js @@ -1,56 +1,67 @@ -'use strict'; +import fsExtra from 'fs-extra'; +import fs from 'fs'; // 'node:fs' +import path from 'path'; // 'node:path' +import { Writable } from 'stream'; // 'node:stream' +import duplexer2 from 'duplexer2'; -module.exports = Extract; +import Parse from './parse.js'; -var Parse = require("../unzip").Parse; -var Writer = require("fstream").Writer; -var Writable = require('readable-stream/writable'); -var path = require('path'); -var inherits = require('util').inherits; +export default function Extract (opts) { + // make sure path is normalized before using it + opts.path = path.resolve(path.normalize(opts.path)); -inherits(Extract, Writable); + const parser = Parse(opts); -function Extract (opts) { - var self = this; - if (!(this instanceof Extract)) { - return new Extract(opts); - } - - Writable.apply(this); - this._opts = opts || { verbose: false }; - - this._parser = Parse(this._opts); - this._parser.on('error', function(err) { - self.emit('error', err); - }); - this.on('finish', function() { - self._parser.end(); + const outStream = new Writable({ + objectMode: true, + write }); - var writer = Writer({ - type: 'Directory', - path: opts.path - }); - writer.on('error', function(err) { - self.emit('error', err); - }); - writer.on('close', function() { - self.emit('close') - }); + async function write(entry, encoding, cb) { + // to avoid zip slip (writing outside of the destination), we resolve + // the target path, and make sure it's nested in the intended + // destination, or not extract it otherwise. + // NOTE: Need to normalize to forward slashes for UNIX OS's to properly + // ignore the zip slipped file entirely + const extractPath = path.join(opts.path, entry.path.replace(/\\/g, '/')); + const rel = path.relative(opts.path, extractPath); + if (rel === '' || rel.startsWith('..') || path.isAbsolute(rel)) { + return cb(); + } - this.on('pipe', function(source) { - if (opts.verbose && source.path) { - console.log('Archive: ', source.path); + if (entry.type == 'Directory') { + await fsExtra.ensureDir(extractPath); + return cb(); } + + await fsExtra.ensureDir(path.dirname(extractPath)); + + const writer = opts.getWriter ? opts.getWriter({path: extractPath}) : fs.createWriteStream(extractPath); + + entry.pipe(writer) + .on('error', cb) + .on('close', cb); + }; + + // Create a combined stream + const extract = duplexer2(parser, outStream); + + parser.once('crx-header', function(crxHeader) { + extract.crxHeader = crxHeader; }); - this._parser.pipe(writer); -} + parser + .pipe(outStream) + .on('finish', function() { + extract.emit('close'); + }); -Extract.prototype._write = function (chunk, encoding, callback) { - if (this._parser.write(chunk)) { - return callback(); - } + extract.promise = function() { + return new Promise(function(resolve, reject) { + extract.on('close', resolve); + extract.on('error', reject); + }); + }; - return this._parser.once('drain', callback); -}; + return extract; +} diff --git a/lib/parse.js b/lib/parse.js index 69fc2c40..d95adbdf 100644 --- a/lib/parse.js +++ b/lib/parse.js @@ -1,96 +1,138 @@ -'use strict'; +import zlib from 'zlib'; // 'node:zlib' +import { PassThrough, pipeline } from 'stream'; // 'node:stream' -module.exports = Parse.create = Parse; +import PullStream from './PullStream.js'; +import NoopStream from './NoopStream.js'; +import BufferStream from './BufferStream.js'; +import parseExtraField from './parseExtraField.js'; +import parseDateTime from './parseDateTime.js'; +import parseBuffer from './parseBuffer.js'; -require("setimmediate"); -var Transform = require('readable-stream/transform'); -var inherits = require('util').inherits; -var zlib = require('zlib'); -var binary = require('binary'); -var PullStream = require('pullstream'); -var MatchStream = require('match-stream'); -var Entry = require('./entry'); +const endDirectorySignature = Buffer.alloc(4); +endDirectorySignature.writeUInt32LE(0x06054b50, 0); -inherits(Parse, Transform); - -function Parse(opts) { - var self = this; - if (!(this instanceof Parse)) { - return new Parse(opts); - } - - Transform.call(this, { lowWaterMark: 0 }); - this._opts = opts || { verbose: false }; - this._hasEntryListener = false; +export default function Parse(opts) { + return new Parser(opts); +} - this._pullStream = new PullStream(); - this._pullStream.on("error", function (e) { - self.emit('error', e); - }); - this._pullStream.once("end", function () { - self._streamEnd = true; +const ParserConstructor = function(opts) { + this._opts = opts; + const self = this; + self.on('finish', function() { + self.emit('end'); + self.emit('close'); }); - this._pullStream.once("finish", function () { - self._streamFinish = true; + self._readRecord().catch(function(e) { + if (!self.__emittedError || self.__emittedError !== e) + self.emit('error', e); }); +}; - this._readRecord(); -} - -Parse.prototype._readRecord = function () { - var self = this; - this._pullStream.pull(4, function (err, data) { - if (err) { - return self.emit('error', err); - } +const _readRecord = function () { + const self = this; - if (data.length === 0) { + return self.pull(4).then(function(data) { + if (data.length === 0) return; - } - var signature = data.readUInt32LE(0); + const signature = data.readUInt32LE(0); + + if (signature === 0x34327243) { + return self._readCrxHeader(); + } if (signature === 0x04034b50) { - self._readFile(); - } else if (signature === 0x02014b50) { - self._readCentralDirectoryFileHeader(); - } else if (signature === 0x06054b50) { - self._readEndOfCentralDirectoryRecord(); - } else { - err = new Error('invalid signature: 0x' + signature.toString(16)); - self.emit('error', err); + return self._readFile(); + } + else if (signature === 0x02014b50) { + self.reachedCD = true; + return self._readCentralDirectoryFileHeader(); + } + else if (signature === 0x06054b50) { + return self._readEndOfCentralDirectoryRecord(); } + else if (self.reachedCD) { + // _readEndOfCentralDirectoryRecord expects the EOCD + // signature to be consumed so set includeEof=true + const includeEof = true; + return self.pull(endDirectorySignature, includeEof).then(function() { + return self._readEndOfCentralDirectoryRecord(); + }); + } + else + self.emit('error', new Error('invalid signature: 0x' + signature.toString(16))); + }).then((function(loop) { + if(loop) { + return self._readRecord(); + } + })); +}; + +const _readCrxHeader = function() { + const self = this; + return self.pull(12).then(function(data) { + self.crxHeader = parseBuffer(data, [ + ['version', 4], + ['pubKeyLength', 4], + ['signatureLength', 4], + ]); + return self.pull(self.crxHeader.pubKeyLength + self.crxHeader.signatureLength); + }).then(function(data) { + self.crxHeader.publicKey = data.slice(0, self.crxHeader.pubKeyLength); + self.crxHeader.signature = data.slice(self.crxHeader.pubKeyLength); + self.emit('crx-header', self.crxHeader); + return true; }); }; -Parse.prototype._readFile = function () { - var self = this; - this._pullStream.pull(26, function (err, data) { - if (err) { - return self.emit('error', err); - } +const _readFile = function () { + const self = this; + return self.pull(26).then(function(data) { + const vars = parseBuffer(data, [ + ['versionsNeededToExtract', 2], + ['flags', 2], + ['compressionMethod', 2], + ['lastModifiedTime', 2], + ['lastModifiedDate', 2], + ['crc32', 4], + ['compressedSize', 4], + ['uncompressedSize', 4], + ['fileNameLength', 2], + ['extraFieldLength', 2], + ]); + + vars.lastModifiedDateTime = parseDateTime(vars.lastModifiedDate, vars.lastModifiedTime); + + if (self.crxHeader) vars.crxHeader = self.crxHeader; + + return self.pull(vars.fileNameLength).then(function(fileNameBuffer) { + const fileName = fileNameBuffer.toString('utf8'); + const entry = new PassThrough(); + let __autodraining = false; + + entry.autodrain = function() { + __autodraining = true; + const draining = entry.pipe(new NoopStream()); + draining.promise = function() { + return new Promise(function(resolve, reject) { + draining.on('finish', resolve); + draining.on('error', reject); + }); + }; + return draining; + }; + + entry.buffer = function() { + return BufferStream(entry); + }; - var vars = binary.parse(data) - .word16lu('versionsNeededToExtract') - .word16lu('flags') - .word16lu('compressionMethod') - .word16lu('lastModifiedTime') - .word16lu('lastModifiedDate') - .word32lu('crc32') - .word32lu('compressedSize') - .word32lu('uncompressedSize') - .word16lu('fileNameLength') - .word16lu('extraFieldLength') - .vars; - - return self._pullStream.pull(vars.fileNameLength, function (err, fileName) { - if (err) { - return self.emit('error', err); - } - fileName = fileName.toString('utf8'); - var entry = new Entry(); entry.path = fileName; + entry.props = {}; entry.props.path = fileName; - entry.type = (vars.compressedSize === 0 && /[\/\\]$/.test(fileName)) ? 'Directory' : 'File'; + entry.props.pathBuffer = fileNameBuffer; + entry.props.flags = { + "isUnicode": (vars.flags & 0x800) != 0 + }; + entry.type = (vars.uncompressedSize === 0 && /[/\\]$/.test(fileName)) ? 'Directory' : 'File'; if (self._opts.verbose) { if (entry.type === 'Directory') { @@ -104,211 +146,172 @@ Parse.prototype._readFile = function () { } } - var hasEntryListener = self._hasEntryListener; - if (hasEntryListener) { - self.emit('entry', entry); - } - - self._pullStream.pull(vars.extraFieldLength, function (err, extraField) { - if (err) { - return self.emit('error', err); - } - if (vars.compressionMethod === 0) { - self._pullStream.pull(vars.compressedSize, function (err, compressedData) { - if (err) { - return self.emit('error', err); - } + return self.pull(vars.extraFieldLength).then(function(extraField) { + const extra = parseExtraField(extraField, vars); - if (hasEntryListener) { - entry.write(compressedData); - entry.end(); - } + entry.vars = vars; + entry.extra = extra; - return self._readRecord(); - }); + if (self._opts.forceStream) { + self.push(entry); } else { - var fileSizeKnown = !(vars.flags & 0x08); + self.emit('entry', entry); - var inflater = zlib.createInflateRaw(); - inflater.on('error', function (err) { - self.emit('error', err); + if (self._readableState.pipesCount || (self._readableState.pipes && self._readableState.pipes.length)) + self.push(entry); + } + + if (self._opts.verbose) + console.log({ + filename:fileName, + vars: vars, + extra: extra }); - if (fileSizeKnown) { - entry.size = vars.uncompressedSize; - if (hasEntryListener) { - entry.on('finish', self._readRecord.bind(self)); - self._pullStream.pipe(vars.compressedSize, inflater).pipe(entry); - } else { - self._pullStream.drain(vars.compressedSize, function (err) { - if (err) { - return self.emit('error', err); - } - self._readRecord(); - }); - } - } else { - var descriptorSig = new Buffer(4); - descriptorSig.writeUInt32LE(0x08074b50, 0); - - var matchStream = new MatchStream({ pattern: descriptorSig }, function (buf, matched, extra) { - if (hasEntryListener) { - if (!matched) { - return this.push(buf); - } - this.push(buf); + const fileSizeKnown = !(vars.flags & 0x08) || vars.compressedSize > 0; + let eof; + + entry.__autodraining = __autodraining; // expose __autodraining for test purposes + const inflater = (vars.compressionMethod && !__autodraining) ? zlib.createInflateRaw() : PassThrough(); + + if (fileSizeKnown) { + entry.size = vars.uncompressedSize; + eof = vars.compressedSize; + } else { + eof = Buffer.alloc(4); + eof.writeUInt32LE(0x08074b50, 0); + } + + return new Promise(function(resolve, reject) { + pipeline( + self.stream(eof), + inflater, + entry, + function (err) { + if (err) { + return reject(err); } - setImmediate(function() { - self._pullStream.unpipe(); - self._pullStream.prepend(extra); - self._processDataDescriptor(entry); - }); - return this.push(null); - }); - - self._pullStream.pipe(matchStream); - if (hasEntryListener) { - matchStream.pipe(inflater).pipe(entry); + + return fileSizeKnown ? resolve(fileSizeKnown) : self._processDataDescriptor(entry).then(resolve).catch(reject); } - } - } + ); + }); }); }); }); }; -Parse.prototype._processDataDescriptor = function (entry) { - var self = this; - this._pullStream.pull(16, function (err, data) { - if (err) { - return self.emit('error', err); - } - - var vars = binary.parse(data) - .word32lu('dataDescriptorSignature') - .word32lu('crc32') - .word32lu('compressedSize') - .word32lu('uncompressedSize') - .vars; +const _processDataDescriptor = function (entry) { + const self = this; + return self.pull(16).then(function(data) { + const vars = parseBuffer(data, [ + ['dataDescriptorSignature', 4], + ['crc32', 4], + ['compressedSize', 4], + ['uncompressedSize', 4], + ]); entry.size = vars.uncompressedSize; - self._readRecord(); + return true; }); }; -Parse.prototype._readCentralDirectoryFileHeader = function () { - var self = this; - this._pullStream.pull(42, function (err, data) { - if (err) { - return self.emit('error', err); - } - - var vars = binary.parse(data) - .word16lu('versionMadeBy') - .word16lu('versionsNeededToExtract') - .word16lu('flags') - .word16lu('compressionMethod') - .word16lu('lastModifiedTime') - .word16lu('lastModifiedDate') - .word32lu('crc32') - .word32lu('compressedSize') - .word32lu('uncompressedSize') - .word16lu('fileNameLength') - .word16lu('extraFieldLength') - .word16lu('fileCommentLength') - .word16lu('diskNumber') - .word16lu('internalFileAttributes') - .word32lu('externalFileAttributes') - .word32lu('offsetToLocalFileHeader') - .vars; - - return self._pullStream.pull(vars.fileNameLength, function (err, fileName) { - if (err) { - return self.emit('error', err); - } - fileName = fileName.toString('utf8'); - - self._pullStream.pull(vars.extraFieldLength, function (err, extraField) { - if (err) { - return self.emit('error', err); - } - self._pullStream.pull(vars.fileCommentLength, function (err, fileComment) { - if (err) { - return self.emit('error', err); - } - return self._readRecord(); - }); +const _readCentralDirectoryFileHeader = function () { + const self = this; + return self.pull(42).then(function(data) { + const vars = parseBuffer(data, [ + ['versionMadeBy', 2], + ['versionsNeededToExtract', 2], + ['flags', 2], + ['compressionMethod', 2], + ['lastModifiedTime', 2], + ['lastModifiedDate', 2], + ['crc32', 4], + ['compressedSize', 4], + ['uncompressedSize', 4], + ['fileNameLength', 2], + ['extraFieldLength', 2], + ['fileCommentLength', 2], + ['diskNumber', 2], + ['internalFileAttributes', 2], + ['externalFileAttributes', 4], + ['offsetToLocalFileHeader', 4], + ]); + + return self.pull(vars.fileNameLength).then(function(fileName) { + vars.fileName = fileName.toString('utf8'); + return self.pull(vars.extraFieldLength); + }) + .then(function() { + return self.pull(vars.fileCommentLength); + }) + .then(function() { + return true; }); - }); }); }; -Parse.prototype._readEndOfCentralDirectoryRecord = function () { - var self = this; - this._pullStream.pull(18, function (err, data) { - if (err) { - return self.emit('error', err); - } +const _readEndOfCentralDirectoryRecord = function() { + const self = this; + return self.pull(18).then(function(data) { + + const vars = parseBuffer(data, [ + ['diskNumber', 2], + ['diskStart', 2], + ['numberOfRecordsOnDisk', 2], + ['numberOfRecords', 2], + ['sizeOfCentralDirectory', 4], + ['offsetToStartOfCentralDirectory', 4], + ['commentLength', 2], + ]); + + return self.pull(vars.commentLength).then(function() { + self.end(); + self.push(null); + }); - var vars = binary.parse(data) - .word16lu('diskNumber') - .word16lu('diskStart') - .word16lu('numberOfRecordsOnDisk') - .word16lu('numberOfRecords') - .word32lu('sizeOfCentralDirectory') - .word32lu('offsetToStartOfCentralDirectory') - .word16lu('commentLength') - .vars; - - if (vars.commentLength) { - setImmediate(function() { - self._pullStream.pull(vars.commentLength, function (err, comment) { - if (err) { - return self.emit('error', err); - } - comment = comment.toString('utf8'); - return self._pullStream.end(); - }); - }); + }); +}; - } else { - self._pullStream.end(); - } +const promise = function() { + const self = this; + return new Promise(function(resolve, reject) { + self.on('finish', resolve); + self.on('error', reject); }); }; -Parse.prototype._transform = function (chunk, encoding, callback) { - if (this._pullStream.write(chunk)) { - return callback(); +class Parser extends PullStream { + constructor(opts = { verbose: false }) { + super(opts); + ParserConstructor.call(this, opts); } - this._pullStream.once('drain', callback); -}; + _readRecord() { + return _readRecord.call(this); + }; -Parse.prototype.pipe = function (dest, opts) { - var self = this; - if (typeof dest.add === "function") { - self.on("entry", function (entry) { - dest.add(entry); - }) - } - return Transform.prototype.pipe.apply(this, arguments); -}; + _readCrxHeader() { + return _readCrxHeader.call(this); + }; -Parse.prototype._flush = function (callback) { - if (!this._streamEnd || !this._streamFinish) { - return setImmediate(this._flush.bind(this, callback)); - } + _readFile() { + return _readFile.call(this); + }; - this.emit('close'); - return callback(); -}; + _processDataDescriptor(entry) { + return _processDataDescriptor.call(this, entry); + }; -Parse.prototype.addListener = function(type, listener) { - if ('entry' === type) { - this._hasEntryListener = true; - } - return Transform.prototype.addListener.call(this, type, listener); -}; + _readCentralDirectoryFileHeader() { + return _readCentralDirectoryFileHeader.call(this); + }; -Parse.prototype.on = Parse.prototype.addListener; + _readEndOfCentralDirectoryRecord() { + return _readEndOfCentralDirectoryRecord.call(this); + }; + + promise() { + return promise.call(this); + }; +} diff --git a/lib/parseBuffer.js b/lib/parseBuffer.js new file mode 100644 index 00000000..f5432da7 --- /dev/null +++ b/lib/parseBuffer.js @@ -0,0 +1,51 @@ +function parseUIntLE(buffer, offset, size) { + let result; + switch(size) { + case 1: + result = buffer.readUInt8(offset); + break; + case 2: + result = buffer.readUInt16LE(offset); + break; + case 4: + result = buffer.readUInt32LE(offset); + break; + case 8: + result = Number(buffer.readBigUInt64LE(offset)); + break; + default: + throw new Error('Unsupported UInt LE size!'); + } + return result; +} + +/** + * Parses sequential unsigned little endian numbers from the head of the passed buffer according to + * the specified format passed. If the buffer is not large enough to satisfy the full format, + * null values will be assigned to the remaining keys. + * @param {*} buffer The buffer to sequentially extract numbers from. + * @param {*} format Expected format to follow when extrcting values from the buffer. A list of list entries + * with the following structure: + * [ + * [ + * , // Name of the key to assign the extracted number to. + * // The size in bytes of the number to extract. possible values are 1, 2, 4, 8. + * ], + * ... + * ] + * @returns An object with keys set to their associated extracted values. + */ +export default function parse(buffer, format) { + const result = {}; + let offset = 0; + for(const [key, size] of format) { + if(buffer.length >= offset + size) { + result[key] = parseUIntLE(buffer, offset, size); + } + else { + result[key] = null; + } + offset += size; + } + return result; +} diff --git a/lib/parseDateTime.js b/lib/parseDateTime.js new file mode 100644 index 00000000..82df0f0f --- /dev/null +++ b/lib/parseDateTime.js @@ -0,0 +1,13 @@ +// Dates in zip file entries are stored as DosDateTime +// Spec is here: https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-dosdatetimetofiletime + +export default function parseDateTime(date, time) { + const day = date & 0x1F; + const month = date >> 5 & 0x0F; + const year = (date >> 9 & 0x7F) + 1980; + const seconds = time ? (time & 0x1F) * 2 : 0; + const minutes = time ? (time >> 5) & 0x3F : 0; + const hours = time ? (time >> 11): 0; + + return new Date(Date.UTC(year, month-1, day, hours, minutes, seconds)); +} \ No newline at end of file diff --git a/lib/parseExtraField.js b/lib/parseExtraField.js new file mode 100644 index 00000000..25841159 --- /dev/null +++ b/lib/parseExtraField.js @@ -0,0 +1,40 @@ +import parseBuffer from './parseBuffer.js'; + +export default function parseExtraField(extraField, vars) { + let extra; + // Find the ZIP64 header, if present. + while(!extra && extraField && extraField.length) { + const candidateExtra = parseBuffer(extraField, [ + ['signature', 2], + ['partSize', 2], + ]); + + if(candidateExtra.signature === 0x0001) { + // parse buffer based on data in ZIP64 central directory; order is important! + const fieldsToExpect = []; + if (vars.uncompressedSize === 0xffffffff) fieldsToExpect.push(['uncompressedSize', 8]); + if (vars.compressedSize === 0xffffffff) fieldsToExpect.push(['compressedSize', 8]); + if (vars.offsetToLocalFileHeader === 0xffffffff) fieldsToExpect.push(['offsetToLocalFileHeader', 8]); + + // slice off the 4 bytes for signature and partSize + extra = parseBuffer(extraField.slice(4), fieldsToExpect); + } else { + // Advance the buffer to the next part. + // The total size of this part is the 4 byte header + partsize. + extraField = extraField.slice(candidateExtra.partSize + 4); + } + } + + extra = extra || {}; + + if (vars.compressedSize === 0xffffffff) + vars.compressedSize = extra.compressedSize; + + if (vars.uncompressedSize === 0xffffffff) + vars.uncompressedSize= extra.uncompressedSize; + + if (vars.offsetToLocalFileHeader === 0xffffffff) + vars.offsetToLocalFileHeader = extra.offsetToLocalFileHeader; + + return extra; +} diff --git a/lib/parseOne.js b/lib/parseOne.js new file mode 100644 index 00000000..10f82b3b --- /dev/null +++ b/lib/parseOne.js @@ -0,0 +1,60 @@ +import { PassThrough, Transform } from 'stream'; // 'node:stream' +import duplexer2 from 'duplexer2'; + +import Parse from './parse.js'; +import BufferStream from './BufferStream.js'; + +export default function parseOne(match, opts) { + const inStream = new PassThrough({objectMode:true}); + const outStream = new PassThrough(); + + const transformer = new Transform({ + objectMode: true, + transform + }); + + const re = match instanceof RegExp ? match : (match && new RegExp(match)); + let found; + + function transform(entry, e, cb) { + if (found || (re && !re.exec(entry.path))) { + entry.autodrain(); + return cb(); + } else { + found = true; + out.emit('entry', entry); + entry.on('error', function(e) { + outStream.emit('error', e); + }); + entry.pipe(outStream) + .on('error', function(err) { + cb(err); + }) + .on('finish', function(d) { + cb(null, d); + }); + } + }; + + inStream.pipe(Parse(opts)) + .on('error', function(err) { + outStream.emit('error', err); + }) + .pipe(transformer) + .on('error', Object) // Silence error as its already addressed in transform + .on('finish', function() { + if (!found) + outStream.emit('error', new Error('PATTERN_NOT_FOUND')); + else + outStream.end(); + }); + + // Create a combined stream + const out = duplexer2(inStream, outStream); + + out.buffer = function() { + return BufferStream(outStream); + }; + + return out; +} diff --git a/package.json b/package.json index 5df6c62f..2396abf6 100644 --- a/package.json +++ b/package.json @@ -1,9 +1,21 @@ { - "name": "unzip", - "version": "0.1.11", - "description": "Unzip cross-platform streaming API compatible with fstream and fs.ReadStream", - "author": "Evan Oxfeld ", - "maintainers": [ + "name": "unzipper", + "version": "0.12.5", + "type": "module", + "main": "./index.cjs", + "exports": { + ".": { + "import": "./index.js", + "require": "./index.cjs" + } + }, + "description": "Unzip cross-platform streaming API ", + "author": "Ziggy Jonsson ", + "contributors": [ + { + "name": "Ziggy Jonsson", + "email": "ziggy.jonsson.nyc@gmail.com" + }, { "name": "Evan Oxfeld", "email": "eoxfeld@gmail.com" @@ -11,26 +23,43 @@ { "name": "Joe Ferner", "email": "joe.ferner@nearinfinity.com" + }, + { + "name": "Nikolay Kuchumov", + "email": "kuchumovn@gmail.com" } ], "repository": { "type": "git", - "url": "https://github.com/EvanOxfeld/node-unzip.git" + "url": "https://github.com/ZJONSSON/node-unzipper.git" }, "license": "MIT", "dependencies": { - "fstream": ">= 0.1.30 < 1", - "pullstream": ">= 0.4.1 < 1", - "binary": ">= 0.3.0 < 1", - "readable-stream": "~1.0.31", - "setimmediate": ">= 1.0.1 < 2", - "match-stream": ">= 0.0.2 < 1" + "bluebird": "~3.7.2", + "duplexer2": "~0.1.4", + "fs-extra": "11.3.1", + "graceful-fs": "^4.2.2", + "node-int64": "^0.4.0" }, "devDependencies": { - "tap": ">= 0.3.0 < 1", - "temp": ">= 0.4.0 < 1", + "@aws-sdk/client-s3": "^3.1066.0", + "@babel/cli": "^7.29.7", + "@babel/core": "^7.29.7", + "@babel/preset-env": "^7.29.7", + "@babel/register": "^7.29.7", + "@eslint/js": "^9.2.0", + "aws-sdk": "^2.1636.0", + "cross-env": "^6.0.3", "dirdiff": ">= 0.0.1 < 1", - "stream-buffers": ">= 0.2.5 < 1" + "eslint": "^9.2.0", + "globals": "^15.2.0", + "iconv-lite": "^0.4.24", + "if-node-version": "^1.1.1", + "request": "^2.88.0", + "rimraf": "^3.0.2", + "stream-buffers": ">= 0.2.5 < 1", + "tap": "^16.3.10", + "temp": ">= 0.4.0 < 1" }, "directories": { "example": "examples", @@ -45,8 +74,21 @@ "stream", "extract" ], - "main": "unzip.js", "scripts": { - "test": "tap ./test/*.js" + "build": "npm run clean-for-build && npm run build-commonjs", + "clean-for-build": "rimraf ./dist", + "build-commonjs-modules": "cross-env BABEL_ENV=commonjs babel ./lib --out-dir ./dist --source-maps", + "build-commonjs-package.json": "node scripts/create-commonjs-package-json.cjs", + "build-commonjs": "npm run build-commonjs-modules && npm run build-commonjs-package.json", + "test": "npm run test-node-above-10 && npm run test-node-10-or-below", + "test-node-above-10": "if-node-version \">10\" npm run test-esm", + "test-node-10-or-below": "if-node-version \"<=10\" npm run test-esm-transpiled", + "test-esm": "npx tap test/*.js --coverage-report=html --lines=90 --functions=85 --statements=90 --branches=80 --reporter=dot", + "test-esm-transpiled": "cross-env BABEL_ENV=commonjs npx tap --node-arg=-r --node-arg=@babel/register test/*.js --coverage-report=html --lines=90 --functions=85 --statements=90 --branches=79.47 --reporter=dot", + "test-commonjs": "npx tap test-commonjs/*.js --coverage-report=html --lines=90 --functions=85 --statements=90 --branches=80 --reporter=dot", + "prepublishOnly": "npm test && npm run build && npm run test-commonjs" + }, + "engines": { + "node": ">=8.0.0" } } diff --git a/scripts/create-commonjs-package-json.cjs b/scripts/create-commonjs-package-json.cjs new file mode 100644 index 00000000..1d3b6969 --- /dev/null +++ b/scripts/create-commonjs-package-json.cjs @@ -0,0 +1,12 @@ +// Creates a `package.json` file in the CommonJS `build` folder. +// That marks that whole folder as CommonJS so that Node.js doesn't complain +// about `require()`-ing those files. + +// import fs from 'fs'; // 'node:fs' +const fs = require('fs'); // 'node:fs' + +fs.writeFileSync('./dist/package.json', JSON.stringify({ + name: 'unzipper/dist', + type: 'commonjs', + private: true +}, null, 2), 'utf8'); diff --git a/test-commonjs/extractFromUrl.js b/test-commonjs/extractFromUrl.js new file mode 100644 index 00000000..b4d3e0ea --- /dev/null +++ b/test-commonjs/extractFromUrl.js @@ -0,0 +1,25 @@ +const test = require("tap").test; +const fs = require("fs"); +const unzipper = require("../index.cjs"); +const os = require("os"); +const request = require("request"); + +test("extract zip from url", function (t) { + const extractPath = os.tmpdir() + "/node-unzip-extract-fromURL"; // Not using path resolve, cause it should be resolved in extract() function + unzipper.Open.url( + request, + "https://github.com/h5bp/html5-boilerplate/releases/download/v7.3.0/html5-boilerplate_v7.3.0.zip" + ) + .then(function(d) { return d.extract({ path: extractPath }); }) + .then(function() { + const dirFiles = fs.readdirSync(extractPath); + const isPassing = + dirFiles.length > 10 && + dirFiles.indexOf("css") > -1 && + dirFiles.indexOf("index.html") > -1 && + dirFiles.indexOf("favicon.ico") > -1; + + t.equal(isPassing, true); + t.end(); + }); +}); diff --git a/test-commonjs/office-files.js b/test-commonjs/office-files.js new file mode 100644 index 00000000..731090c7 --- /dev/null +++ b/test-commonjs/office-files.js @@ -0,0 +1,16 @@ +const test = require('tap').test; +const unzipper = require('../index.cjs'); + +test("get content a docx file without errors", async function () { + const archive = './testData/office/testfile.docx'; + + const directory = await unzipper.Open.file(archive); + await Promise.all(directory.files.map(file => file.buffer())); +}); + +test("get content a xlsx file without errors", async function () { + const archive = './testData/office/testfile.xlsx'; + + const directory = await unzipper.Open.file(archive); + await Promise.all(directory.files.map(file => file.buffer())); +}); diff --git a/test-commonjs/package.json b/test-commonjs/package.json new file mode 100644 index 00000000..87d71c30 --- /dev/null +++ b/test-commonjs/package.json @@ -0,0 +1,5 @@ +{ + "name": "unzipper/test-commonjs", + "type": "commonjs", + "private": true +} diff --git a/test-commonjs/parseOneEntry.js b/test-commonjs/parseOneEntry.js new file mode 100644 index 00000000..6e026635 --- /dev/null +++ b/test-commonjs/parseOneEntry.js @@ -0,0 +1,50 @@ +'use strict'; + +const test = require('tap').test; +const fs = require('fs'); +const streamBuffers = require("stream-buffers"); +const unzipper = require('../index.cjs'); + +const archive = './testData/compressed-standard/archive.zip'; + +test("pipe a single file entry out of a zip", function (t) { + const writableStream = new streamBuffers.WritableStreamBuffer(); + writableStream.on('close', function () { + const str = writableStream.getContentsAsString('utf8'); + const fileStr = fs.readFileSync('./testData/compressed-standard/inflated/file.txt', 'utf8'); + t.equal(str, fileStr); + t.end(); + }); + + fs.createReadStream(archive) + .pipe(unzipper.ParseOne('file.txt')) + .pipe(writableStream); +}); + +test('errors if file is not found', function (t) { + fs.createReadStream(archive) + .pipe(unzipper.ParseOne('not_exists')) + .on('error', function(e) { + t.equal(e.message, 'PATTERN_NOT_FOUND'); + t.end(); + }); +}); + +test('error - invalid signature', function(t) { + unzipper.ParseOne() + .on('error', function(e) { + t.equal(e.message.indexOf('invalid signature'), 0); + t.end(); + }) + .end('this is not a zip file'); +}); + +test('error - file ended', function(t) { + unzipper.ParseOne() + .on('error', function(e) { + if (e.message == 'PATTERN_NOT_FOUND') return; + t.equal(e.message, 'FILE_ENDED'); + t.end(); + }) + .end('t'); +}); diff --git a/test-commonjs/parsePromise.js b/test-commonjs/parsePromise.js new file mode 100644 index 00000000..89bdd517 --- /dev/null +++ b/test-commonjs/parsePromise.js @@ -0,0 +1,49 @@ +'use strict'; + +const test = require('tap').test; +const fs = require('fs'); +const unzipper = require('../index.cjs'); + +let entryRead; + +test("promise should resolve when entries have been processed", function (t) { + const archive = './testData/compressed-standard/archive.zip'; + + fs.createReadStream(archive) + .pipe(unzipper.Parse()) + .on('entry', function(entry) { + if (entry.path !== 'file.txt') + return entry.autodrain(); + + entry.buffer() + .then(function() { + entryRead = true; + }); + }) + .promise() + .then(function() { + t.equal(entryRead, true); + t.end(); + }, function() { + t.fail('This project should resolve'); + t.end(); + }); +}); + +test("promise should be rejected if there is an error in the stream", function (t) { + const archive = './testData/compressed-standard/archive.zip'; + + fs.createReadStream(archive) + .pipe(unzipper.Parse()) + .on('entry', function() { + this.emit('error', new Error('this is an error')); + }) + .promise() + .then(function() { + t.fail('This promise should be rejected'); + t.end(); + }, function(e) { + t.equal(e.message, 'this is an error'); + t.end(); + }); +}); diff --git a/test/autodrain-passthrough.js b/test/autodrain-passthrough.js new file mode 100644 index 00000000..fef678dd --- /dev/null +++ b/test/autodrain-passthrough.js @@ -0,0 +1,45 @@ +import { test } from 'tap'; +import fs from 'fs'; +import { Parse } from '../index.js'; + +test("verify that immediate autodrain does not unzip", function (t) { + const archive = './testData/compressed-standard/archive.zip'; + + fs.createReadStream(archive) + .pipe(Parse()) + .on('entry', function(entry) { + entry.autodrain() + .on('finish', function() { + t.equal(entry.__autodraining, true); + }); + }) + .on('finish', function() { + t.end(); + }); +}); + +test("verify that autodrain promise works", function (t) { + const archive = './testData/compressed-standard/archive.zip'; + + fs.createReadStream(archive) + .pipe(Parse()) + .on('entry', function(entry) { + entry.autodrain() + .promise() + .then(function() { + t.equal(entry.__autodraining, true); + }); + }) + .on('finish', function() { + t.end(); + }); +}); + +test("verify that autodrain resolves after it has finished", function (t) { + const archive = './testData/compressed-standard/archive.zip'; + + fs.createReadStream(archive) + .pipe(Parse()) + .on('entry', entry => entry.autodrain()) + .on('end', () => t.end()); +}); diff --git a/test/broken.js b/test/broken.js new file mode 100644 index 00000000..7764ae69 --- /dev/null +++ b/test/broken.js @@ -0,0 +1,40 @@ +import { test } from 'tap'; +import fs from 'fs'; +import temp from 'temp'; +import { Parse, Extract } from '../index.js'; + + +test("Parse a broken zipfile", function (t) { + const archive = './testData/compressed-standard/broken.zip'; + + fs.createReadStream(archive) + .pipe(Parse()) + .on('entry', function(entry) { + return entry.autodrain(); + }) + .promise() + .catch(function(e) { + t.same(e.message, 'FILE_ENDED'); + t.end(); + }); +}); + + +test("extract a broken", function (t) { + const archive = './testData/compressed-standard/broken.zip'; + + temp.mkdir('node-unzip-', function (err, dirPath) { + if (err) { + throw err; + } + const unzipExtractor = Extract({ path: dirPath }); + + fs.createReadStream(archive) + .pipe(unzipExtractor) + .promise() + .catch(function(e) { + t.same(e.message, 'FILE_ENDED'); + t.end(); + }); + }); +}); \ No newline at end of file diff --git a/test/chunkBoundary.js b/test/chunkBoundary.js new file mode 100644 index 00000000..6fd92fe7 --- /dev/null +++ b/test/chunkBoundary.js @@ -0,0 +1,21 @@ +import { test } from 'tap'; +import fs from 'fs'; +import { Parse } from '../index.js'; + +test("parse an archive that has a file that falls on a chunk boundary", { + timeout: 2000, +}, function (t) { + + + const archive = './testData/chunk-boundary/chunk-boundary-archive.zip'; + + // Use an artificially low highWaterMark to make the edge case more likely to happen. + fs.createReadStream(archive, { highWaterMark: 3 }) + .pipe(Parse()) + .on('entry', function(entry) { + return entry.autodrain(); + }).on("finish", function() { + t.ok(true, 'file complete'); + t.end(); + }); +}); \ No newline at end of file diff --git a/test/compressed-crx.js b/test/compressed-crx.js new file mode 100644 index 00000000..cfa8e4ae --- /dev/null +++ b/test/compressed-crx.js @@ -0,0 +1,75 @@ +import { test } from 'tap'; +import fs from 'fs'; +import temp from 'temp'; +import dirdiff from 'dirdiff'; +import AWS from 'aws-sdk'; +import { Extract, Open } from '../index.js'; + +test('parse/extract crx archive', function (t) { + const archive = './testData/compressed-standard-crx/archive.crx'; + + temp.mkdir('node-unzip-', function (err, dirPath) { + if (err) { + throw err; + } + const unzipExtractor = Extract({ path: dirPath }); + unzipExtractor.on('error', function(err) { + throw err; + }); + unzipExtractor.on('close', testExtractionResults); + + fs.createReadStream(archive).pipe(unzipExtractor); + + function testExtractionResults() { + t.same(unzipExtractor.crxHeader.version, 2); + dirdiff('./testData/compressed-standard-crx/inflated', dirPath, { + fileContents: true + }, function (err, diffs) { + if (err) { + throw err; + } + t.equal(diffs.length, 0, 'extracted directory contents'); + t.end(); + }); + } + }); +}); + +test('open methods', async function(t) { + const archive = './testData/compressed-standard-crx/archive.crx'; + const buffer = fs.readFileSync(archive); + const s3 = new AWS.S3({region: 'us-east-1'}); + + // We have to modify the `getObject` and `headObject` to use makeUnauthenticated + s3.getObject = function(params, cb) { + return s3.makeUnauthenticatedRequest('getObject', params, cb); + }; + + s3.headObject = function(params, cb) { + return s3.makeUnauthenticatedRequest('headObject', params, cb); + }; + + const tests = [ + {name: 'buffer', args: [buffer]}, + {name: 'file', args: [archive]}, + // {name: 'url', args: [request, 'https://s3.amazonaws.com/unzipper/archive.crx']}, + // {name: 's3', args: [s3, { Bucket: 'unzipper', Key: 'archive.crx'}]} + ]; + + for (const test of tests) { + t.test(test.name, async function(t) { + t.test('opening with crx option', function(t) { + const method = Open[test.name]; + method.apply(method, test.args.concat({crx:true})) + .then(function(d) { + return d.files[1].buffer(); + }) + .then(function(d) { + t.same(String(d), '42\n', test.name + ' content matches'); + t.end(); + }); + }); + }); + }; + t.end(); +}); diff --git a/test/compressed.js b/test/compressed.js index 4f598c7c..797e22bb 100644 --- a/test/compressed.js +++ b/test/compressed.js @@ -1,16 +1,15 @@ -'use strict'; - -var test = require('tap').test; -var fs = require('fs'); -var path = require('path'); -var temp = require('temp'); -var dirdiff = require('dirdiff'); -var unzip = require('../'); +import { test } from 'tap'; +import fs from 'fs'; +import path from 'path'; +import temp from 'temp'; +import dirdiff from 'dirdiff'; +import { Parse, Extract } from '../index.js'; +import il from 'iconv-lite'; test("parse compressed archive (created by POSIX zip)", function (t) { - var archive = path.join(__dirname, '../testData/compressed-standard/archive.zip'); + const archive = './testData/compressed-standard/archive.zip'; - var unzipParser = unzip.Parse(); + const unzipParser = Parse(); fs.createReadStream(archive).pipe(unzipParser); unzipParser.on('error', function(err) { throw err; @@ -19,14 +18,30 @@ test("parse compressed archive (created by POSIX zip)", function (t) { unzipParser.on('close', t.end.bind(this)); }); +test("parse compressed archive (created by DOS zip)", function (t) { + const archive = './testData/compressed-cp866/archive.zip'; + + const unzipParser = Parse(); + fs.createReadStream(archive).pipe(unzipParser); + unzipParser.on('entry', function(entry) { + const fileName = entry.props.flags.isUnicode ? entry.path : il.decode(entry.props.pathBuffer, 'cp866'); + t.equal(fileName, 'Тест.txt'); + }); + unzipParser.on('error', function(err) { + throw err; + }); + + unzipParser.on('close', t.end.bind(this)); +}); + test("extract compressed archive w/ file sizes known prior to zlib inflation (created by POSIX zip)", function (t) { - var archive = path.join(__dirname, '../testData/compressed-standard/archive.zip'); + const archive = './testData/compressed-standard/archive.zip'; temp.mkdir('node-unzip-', function (err, dirPath) { if (err) { throw err; } - var unzipExtractor = unzip.Extract({ path: dirPath }); + const unzipExtractor = Extract({ path: dirPath }); unzipExtractor.on('error', function(err) { throw err; }); @@ -34,8 +49,15 @@ test("extract compressed archive w/ file sizes known prior to zlib inflation (cr fs.createReadStream(archive).pipe(unzipExtractor); - function testExtractionResults() { - dirdiff(path.join(__dirname, '../testData/compressed-standard/inflated'), dirPath, { + async function testExtractionResults() { + const root = './testData/compressed-standard/inflated'; + + // since empty directories can not be checked into git we have to + // create them + await fs.promises.mkdir(path.resolve(root, 'emptydir'), { recursive: true }); + await fs.promises.mkdir(path.resolve(root, 'emptyroot/emptydir'), { recursive: true }); + + dirdiff('./testData/compressed-standard/inflated', dirPath, { fileContents: true }, function (err, diffs) { if (err) { diff --git a/test/extract-finish.js b/test/extract-finish.js new file mode 100644 index 00000000..37c1e040 --- /dev/null +++ b/test/extract-finish.js @@ -0,0 +1,47 @@ +import { test } from 'tap'; +import fs from 'fs'; +import os from 'os'; +import temp from 'temp'; +import { Extract } from '../index.js'; +import Stream from 'stream'; + + +test("Only emit finish/close when extraction has completed", function (t) { + const archive = './testData/compressed-standard/archive.zip'; + + temp.mkdir('node-unzip-finish-', function (err) { + if (err) { + throw err; + } + + let filesDone = 0; + + function getWriter() { + const delayStream = new Stream.Transform(); + + delayStream._transform = function(d, e, cb) { + setTimeout(cb, 500); + }; + + delayStream._flush = function(cb) { + filesDone += 1; + setTimeout(cb, 100); + delayStream.emit('close'); + }; + + return delayStream; + } + + + const unzipExtractor = Extract({ getWriter: getWriter, path: os.tmpdir() }); + unzipExtractor.on('error', function(err) { + throw err; + }); + unzipExtractor.promise().then(function() { + t.same(filesDone, 2); + t.end(); + }); + + fs.createReadStream(archive).pipe(unzipExtractor); + }); +}); \ No newline at end of file diff --git a/test/extractFromUrl.js b/test/extractFromUrl.js new file mode 100644 index 00000000..e7cc12b5 --- /dev/null +++ b/test/extractFromUrl.js @@ -0,0 +1,25 @@ +import { test } from "tap"; +import fs from "fs"; +import { Open } from "../index.js"; +import os from "os"; +import request from "request"; + +test("extract zip from url", function (t) { + const extractPath = os.tmpdir() + "/node-unzip-extract-fromURL"; // Not using path resolve, cause it should be resolved in extract() function + Open.url( + request, + "https://github.com/h5bp/html5-boilerplate/releases/download/v7.3.0/html5-boilerplate_v7.3.0.zip" + ) + .then(function(d) { return d.extract({ path: extractPath }); }) + .then(function() { + const dirFiles = fs.readdirSync(extractPath); + const isPassing = + dirFiles.length > 10 && + dirFiles.indexOf("css") > -1 && + dirFiles.indexOf("index.html") > -1 && + dirFiles.indexOf("favicon.ico") > -1; + + t.equal(isPassing, true); + t.end(); + }); +}); diff --git a/test/extractNormalize.js b/test/extractNormalize.js new file mode 100644 index 00000000..021aa020 --- /dev/null +++ b/test/extractNormalize.js @@ -0,0 +1,89 @@ +import { test } from 'tap'; +import fs from 'fs'; +import os from 'os'; +import temp from 'temp'; +import { Extract } from '../index.js'; +import Stream from 'stream'; + + +test("Extract should normalize the path option", function (t) { + const archive = './testData/compressed-standard/archive.zip'; + + temp.mkdir('node-unzip-normalize-', function (err) { + if (err) { + throw err; + } + + let filesDone = 0; + + function getWriter() { + const delayStream = new Stream.Transform(); + + delayStream._transform = function(d, e, cb) { + setTimeout(cb, 500); + }; + + delayStream._flush = function(cb) { + filesDone += 1; + cb(); + delayStream.emit('close'); + }; + + return delayStream; + } + + // don't use path.join, it will normalize the path which defeats + // the purpose of this test + const extractPath = os.tmpdir() + "/unzipper\\normalize/././extract\\test"; + + const unzipExtractor = Extract({ getWriter: getWriter, path: extractPath }); + unzipExtractor.on('error', function(err) { + throw err; + }); + unzipExtractor.on('close', function() { + t.same(filesDone, 2); + t.end(); + }); + + fs.createReadStream(archive).pipe(unzipExtractor); + }); +}); + +test("Extract should resolve after normalize the path option", function (t) { + const archive = './testData/compressed-standard/archive.zip'; + + temp.mkdir('node-unzip-normalize-2-', function (err) { + if (err) { + throw err; + } + + let filesDone = 0; + + function getWriter() { + const delayStream = new Stream.Transform(); + + delayStream._transform = function(d, e, cb) { + setTimeout(cb, 500); + }; + + delayStream._flush = function(cb) { + filesDone += 1; + cb(); + delayStream.emit('close'); + }; + + return delayStream; + } + + const unzipExtractor = Extract({ getWriter: getWriter, path: '.' }); + unzipExtractor.on('error', function(err) { + throw err; + }); + unzipExtractor.on('close', function() { + t.same(filesDone, 2); + t.end(); + }); + + fs.createReadStream(archive).pipe(unzipExtractor); + }); +}); \ No newline at end of file diff --git a/test/fileSizeUnknownFlag.js b/test/fileSizeUnknownFlag.js index a0f3af4f..3634976d 100644 --- a/test/fileSizeUnknownFlag.js +++ b/test/fileSizeUnknownFlag.js @@ -1,16 +1,13 @@ -'use strict'; - -var test = require('tap').test; -var fs = require('fs'); -var path = require('path'); -var temp = require('temp'); -var dirdiff = require('dirdiff'); -var unzip = require('../'); +import { test } from 'tap'; +import fs from 'fs'; +import temp from 'temp'; +import dirdiff from 'dirdiff'; +import { Parse, Extract } from '../index.js'; test("parse archive w/ file size unknown flag set (created by OS X Finder)", function (t) { - var archive = path.join(__dirname, '../testData/compressed-OSX-Finder/archive.zip'); + const archive = './testData/compressed-OSX-Finder/archive.zip'; - var unzipParser = unzip.Parse(); + const unzipParser = Parse(); fs.createReadStream(archive).pipe(unzipParser); unzipParser.on('error', function(err) { throw err; @@ -20,13 +17,13 @@ test("parse archive w/ file size unknown flag set (created by OS X Finder)", fun }); test("extract archive w/ file size unknown flag set (created by OS X Finder)", function (t) { - var archive = path.join(__dirname, '../testData/compressed-OSX-Finder/archive.zip'); + const archive = './testData/compressed-OSX-Finder/archive.zip'; temp.mkdir('node-unzip-', function (err, dirPath) { if (err) { throw err; } - var unzipExtractor = unzip.Extract({ path: dirPath }); + const unzipExtractor = Extract({ path: dirPath }); unzipExtractor.on('error', function(err) { throw err; }); @@ -35,7 +32,7 @@ test("extract archive w/ file size unknown flag set (created by OS X Finder)", f fs.createReadStream(archive).pipe(unzipExtractor); function testExtractionResults() { - dirdiff(path.join(__dirname, '../testData/compressed-OSX-Finder/inflated'), dirPath, { + dirdiff('./testData/compressed-OSX-Finder/inflated', dirPath, { fileContents: true }, function (err, diffs) { if (err) { @@ -49,13 +46,13 @@ test("extract archive w/ file size unknown flag set (created by OS X Finder)", f }); test("archive w/ language encoding flag set", function (t) { - var archive = path.join(__dirname, '../testData/compressed-flags-set/archive.zip'); + const archive = './testData/compressed-flags-set/archive.zip'; temp.mkdir('node-unzip-', function (err, dirPath) { if (err) { throw err; } - var unzipExtractor = unzip.Extract({ path: dirPath }); + const unzipExtractor = Extract({ path: dirPath }); unzipExtractor.on('error', function(err) { throw err; }); @@ -64,7 +61,7 @@ test("archive w/ language encoding flag set", function (t) { fs.createReadStream(archive).pipe(unzipExtractor); function testExtractionResults() { - dirdiff(path.join(__dirname, '../testData/compressed-flags-set/inflated'), dirPath, { + dirdiff('./testData/compressed-flags-set/inflated', dirPath, { fileContents: true }, function (err, diffs) { if (err) { diff --git a/test/forceStream.js b/test/forceStream.js new file mode 100644 index 00000000..ac2e4d5d --- /dev/null +++ b/test/forceStream.js @@ -0,0 +1,25 @@ +import { test } from 'tap'; +import fs from 'fs'; +import Stream from 'stream'; +import { Parse } from '../index.js'; + +test("verify that setting the forceStream option emits a data event instead of entry", function (t) { + const archive = './testData/compressed-standard/archive.zip'; + + let dataEventEmitted = false; + let entryEventEmitted = false; + fs.createReadStream(archive) + .pipe(Parse({ forceStream: true })) + .on('data', function(entry) { + t.equal(entry instanceof Stream.PassThrough, true); + dataEventEmitted = true; + }) + .on('entry', function() { + entryEventEmitted = true; + }) + .on('finish', function() { + t.equal(dataEventEmitted, true); + t.equal(entryEventEmitted, false); + t.end(); + }); +}); diff --git a/test/notArchive.js b/test/notArchive.js new file mode 100644 index 00000000..30bd535b --- /dev/null +++ b/test/notArchive.js @@ -0,0 +1,50 @@ +import { test } from 'tap'; +import fs from 'fs'; +import temp from 'temp'; +import { Parse, Extract, Open } from '../index.js'; + +const archive = './package.json'; + +test('parse a file that is not an archive', function (t) { + + const unzipParser = Parse(); + fs.createReadStream(archive).pipe(unzipParser); + unzipParser.on('error', function(err) { + t.ok(err.message.indexOf('invalid signature: 0x') !== -1); + t.end(); + }); + + unzipParser.on('close', function(d) { + t.fail('Archive was parsed', d); + }); +}); + +test('extract a file that is not an archive', function (t) { + + temp.mkdir('node-unzip-', function(err, dirPath) { + if (err) { + throw err; + } + const unzipExtractor = Extract({ path: dirPath }); + unzipExtractor.on('error', function(err) { + t.ok(err.message.indexOf('invalid signature: 0x') !== -1); + t.end(); + }); + unzipExtractor.on('close', function() { + t.fail('Archive was extracted'); + }); + + fs.createReadStream(archive).pipe(unzipExtractor); + }); +}); + +test('get content of a single file entry out of a file that is not an archive', function (t) { + Open.file(archive) + .then(function(d) { + t.fail('Archive was opened', d); + }) + .catch(function(err) { + t.equal(err.message, 'FILE_ENDED'); + t.end(); + }); +}); \ No newline at end of file diff --git a/test/office-files.js b/test/office-files.js new file mode 100644 index 00000000..d536c280 --- /dev/null +++ b/test/office-files.js @@ -0,0 +1,16 @@ +import { test } from 'tap'; +import { Open } from '../index.js'; + +test("get content a docx file without errors", async function () { + const archive = './testData/office/testfile.docx'; + + const directory = await Open.file(archive); + await Promise.all(directory.files.map(file => file.buffer())); +}); + +test("get content a xlsx file without errors", async function () { + const archive = './testData/office/testfile.xlsx'; + + const directory = await Open.file(archive); + await Promise.all(directory.files.map(file => file.buffer())); +}); diff --git a/test/open-extract.js b/test/open-extract.js new file mode 100644 index 00000000..eb73485e --- /dev/null +++ b/test/open-extract.js @@ -0,0 +1,39 @@ +import { test } from 'tap'; +import path from 'path'; +import temp from 'temp'; +import dirdiff from 'dirdiff'; +import { Open } from '../index.js'; +import fs from 'fs'; + + +test("extract compressed archive with open.file.extract", function (t) { + const archive = './testData/compressed-standard/archive.zip'; + + temp.mkdir('node-unzip-2', function (err, dirPath) { + if (err) { + throw err; + } + Open.file(archive) + .then(function(d) { + return d.extract({path: dirPath}); + }) + .then(async function() { + const root = './testData/compressed-standard/inflated'; + + // since empty directories can not be checked into git we have to + // create them + await fs.promises.mkdir(path.resolve(root, 'emptydir'), { recursive: true }); + await fs.promises.mkdir(path.resolve(root, 'emptyroot/emptydir'), { recursive: true }); + + dirdiff('./testData/compressed-standard/inflated', dirPath, { + fileContents: true + }, function (err, diffs) { + if (err) { + throw err; + } + t.equal(diffs.length, 0, 'extracted directory contents'); + t.end(); + }); + }); + }); +}); \ No newline at end of file diff --git a/test/openBuffer.js b/test/openBuffer.js new file mode 100644 index 00000000..ac26a490 --- /dev/null +++ b/test/openBuffer.js @@ -0,0 +1,22 @@ +import { test } from 'tap'; +import fs from 'fs'; +import { Open } from '../index.js'; + +test("get content of a single file entry out of a buffer", function (t) { + const archive = './testData/compressed-standard/archive.zip'; + const buffer = fs.readFileSync(archive); + + return Open.buffer(buffer) + .then(function(d) { + const file = d.files.filter(function(file) { + return file.path == 'file.txt'; + })[0]; + + return file.buffer() + .then(function(str) { + const fileStr = fs.readFileSync('./testData/compressed-standard/inflated/file.txt', 'utf8'); + t.equal(str.toString(), fileStr); + t.end(); + }); + }); +}); \ No newline at end of file diff --git a/test/openComment.js b/test/openComment.js new file mode 100644 index 00000000..43655bb5 --- /dev/null +++ b/test/openComment.js @@ -0,0 +1,13 @@ +import { test } from 'tap'; +import { Open } from '../index.js'; + + +test("get comment out of a zip", function (t) { + const archive = './testData/compressed-comment/archive.zip'; + + Open.file(archive) + .then(function(d) { + t.equal('Zipfile has a comment', d.comment); + t.end(); + }); +}); \ No newline at end of file diff --git a/test/openCustom.js b/test/openCustom.js new file mode 100644 index 00000000..ce98aaa4 --- /dev/null +++ b/test/openCustom.js @@ -0,0 +1,37 @@ +import { test } from 'tap'; +import fs from 'fs'; +import { Open } from '../index.js'; + +test("get content of a single file entry out of a zip", function (t) { + const archive = './testData/compressed-standard/archive.zip'; + + const customSource = { + stream: function(offset, length) { + return fs.createReadStream(archive, {start: offset, end: length && offset+length}); + }, + size: function() { + return new Promise(function(resolve, reject) { + fs.stat(archive, function(err, d) { + if (err) + reject(err); + else + resolve(d.size); + }); + }); + } + }; + + return Open.custom(customSource) + .then(function(d) { + const file = d.files.filter(function(file) { + return file.path == 'file.txt'; + })[0]; + + return file.buffer() + .then(function(str) { + const fileStr = fs.readFileSync('./testData/compressed-standard/inflated/file.txt', 'utf8'); + t.equal(str.toString(), fileStr); + t.end(); + }); + }); +}); \ No newline at end of file diff --git a/test/openFile.js b/test/openFile.js new file mode 100644 index 00000000..ddf286fb --- /dev/null +++ b/test/openFile.js @@ -0,0 +1,62 @@ +'use strict'; + +import { test } from 'tap'; +import fs from 'fs'; +import { Open } from '../index.js'; +import il from 'iconv-lite'; + +test("get content of a single file entry out of a zip", function (t) { + const archive = './testData/compressed-standard/archive.zip'; + + return Open.file(archive) + .then(function(d) { + const file = d.files.filter(function(file) { + return file.path == 'file.txt'; + })[0]; + + return file.buffer() + .then(function(str) { + const fileStr = fs.readFileSync('./testData/compressed-standard/inflated/file.txt', 'utf8'); + t.equal(str.toString(), fileStr); + t.end(); + }); + }); +}); + +test("get content of a single file entry out of a DOS zip", function (t) { + const archive = './testData/compressed-cp866/archive.zip'; + + return Open.file(archive, { fileNameEncoding: 'cp866' }) + .then(function(d) { + const file = d.files.filter(function(file) { + const fileName = file.isUnicode ? file.path : il.decode(file.pathBuffer, 'cp866'); + return fileName == 'Тест.txt'; + })[0]; + + return file.buffer() + .then(function(str) { + const fileStr = il.decode(fs.readFileSync('./testData/compressed-cp866/inflated/Тест.txt'), 'cp1251'); + const zipStr = il.decode(str, 'cp1251'); + t.equal(zipStr, fileStr); + t.equal(zipStr, 'Тестовый файл'); + t.end(); + }); + }); +}); + + +test("get multiple buffers concurrently", function (t) { + const archive = './testData/compressed-directory-entry/archive.zip'; + return Open.file(archive) + .then(function(directory) { + return Promise.all(directory.files.map(function(file) { + return file.buffer(); + })) + .then(function(b) { + directory.files.forEach(function(file, i) { + t.equal(file.uncompressedSize, b[i].length); + }); + t.end(); + }); + }); +}); \ No newline at end of file diff --git a/test/openFileEncrypted.js b/test/openFileEncrypted.js new file mode 100644 index 00000000..de45c932 --- /dev/null +++ b/test/openFileEncrypted.js @@ -0,0 +1,59 @@ +import { test } from 'tap'; +import fs from 'fs'; +import { Open } from '../index.js'; + +const archive = './testData/compressed-encrypted/archive.zip'; + +test("get content of a single file entry out of a zip", function (t) { + return Open.file(archive) + .then(function(d) { + const file = d.files.filter(function(file) { + return file.path == 'file.txt'; + })[0]; + + return file.buffer('abc123') + .then(function(str) { + const fileStr = fs.readFileSync('./testData/compressed-standard/inflated/file.txt', 'utf8'); + t.equal(str.toString(), fileStr); + t.end(); + }); + }); +}); + +test("error if password is missing", function (t) { + return Open.file(archive) + .then(function(d) { + const file = d.files.filter(function(file) { + return file.path == 'file.txt'; + })[0]; + + return file.buffer() + .then(function() { + t.error('should error'); + }, function(e) { + t.equal(e.message, 'MISSING_PASSWORD'); + }) + .then(function() { + t.end(); + }); + }); +}); + +test("error if password is wrong", function (t) { + return Open.file(archive) + .then(function(d) { + const file = d.files.filter(function(file) { + return file.path == 'file.txt'; + })[0]; + + return file.buffer('abc1') + .then(function() { + t.error('should error'); + }, function(e) { + t.equal(e.message, 'BAD_PASSWORD'); + }) + .then(function() { + t.end(); + }); + }); +}); \ No newline at end of file diff --git a/test/openS3.js b/test/openS3.js new file mode 100644 index 00000000..fe24ec3d --- /dev/null +++ b/test/openS3.js @@ -0,0 +1,31 @@ +import { test } from 'tap'; +import fs from 'fs'; +import { Open } from '../index.js'; +import AWS from 'aws-sdk'; +const s3 = new AWS.S3({region: 'us-east-1'}); + + +// We have to modify the `getObject` and `headObject` to use makeUnauthenticated +s3.getObject = function(params, cb) { + return s3.makeUnauthenticatedRequest('getObject', params, cb); +}; + +s3.headObject = function(params, cb) { + return s3.makeUnauthenticatedRequest('headObject', params, cb); +}; + +test("get content of a single file entry out of a zip", { skip: true }, function(t) { + return Open.s3(s3, { Bucket: 'unzipper', Key: 'archive.zip' }) + .then(function(d) { + const file = d.files.filter(function(file) { + return file.path == 'file.txt'; + })[0]; + + return file.buffer() + .then(function(str) { + const fileStr = fs.readFileSync('./testData/compressed-standard/inflated/file.txt', 'utf8'); + t.equal(str.toString(), fileStr); + t.end(); + }); + }); +}); diff --git a/test/openS3_v3.js b/test/openS3_v3.js new file mode 100644 index 00000000..c0efee44 --- /dev/null +++ b/test/openS3_v3.js @@ -0,0 +1,58 @@ +import { test } from "tap"; +import fs from "fs"; +import Stream from "stream"; // "node:stream" +import { Open } from "../index.js"; + +const version = +process.version.replace("v", "").split(".")[0]; + +function createS3ClientMock(buffer) { + return { + send: async function(command) { + if (command.constructor.name == "HeadObjectCommand") { + return { + ContentLength: buffer.length, + }; + } + + if (command.constructor.name == "GetObjectCommand") { + const match = command.input.Range.match(/^bytes=(\d+)-(\d*)$/); + const offset = Number(match[1]); + const end = match[2] ? Number(match[2]) + 1 : undefined; + const stream = Stream.PassThrough(); + + stream.end(buffer.slice(offset, end)); + + return { + Body: stream, + }; + } + + throw new Error("Unexpected command: " + command.constructor.name); + }, + }; +} + +test( + "get content of a single file entry out of a zip", + { skip: version < 16 }, + function (t) { + const archive = "./testData/compressed-standard/archive.zip"; + const buffer = fs.readFileSync(archive); + const client = createS3ClientMock(buffer); + + return Open.s3_v3(client, { + Bucket: "test", + Key: "archive.zip", + }).then(function (d) { + const file = d.files.filter(function (file) { + return file.path == "file.txt"; + })[0]; + + return file.buffer().then(function (str) { + const fileStr = fs.readFileSync("./testData/compressed-standard/inflated/file.txt", "utf8"); + t.equal(str.toString(), fileStr); + t.end(); + }); + }); + } +); diff --git a/test/openUrl.js b/test/openUrl.js new file mode 100644 index 00000000..f3037b9f --- /dev/null +++ b/test/openUrl.js @@ -0,0 +1,19 @@ +import { test } from 'tap'; +import fs from 'fs'; +import { Open } from '../index.js'; +import request from 'request'; + +test("get content of a single file entry out of a 502 MB zip from web", function (t) { + return Open.url(request, 'https://github.com/twbs/bootstrap/releases/download/v4.0.0/bootstrap-4.0.0-dist.zip') + .then(function(d) { + const file = d.files.filter(function(d) { + return d.path === 'css/bootstrap-reboot.min.css'; + })[0]; + return file.buffer(); + }) + .then(function(str) { + const fileStr = fs.readFileSync('./testData/bootstrap-reboot.min.css', 'utf8'); + t.equal(str.toString(), fileStr); + t.end(); + }); +}); \ No newline at end of file diff --git a/test/parseBuffer.js b/test/parseBuffer.js new file mode 100644 index 00000000..6bd705ab --- /dev/null +++ b/test/parseBuffer.js @@ -0,0 +1,70 @@ +'use strict'; + +import { test } from 'tap'; +import parseBuffer from '../lib/parseBuffer.js'; + +const buf = Buffer.from([ + 0x62, + 0x75, + 0x66, + 0x68, + 0x65, + 0x72, + 0xFF, + 0xAE, + 0x00, + 0x11, + 0x99, + 0xD7, + 0x7B, + 0x13, + 0x35 +]); + +test(`parse little endian values for increasing byte size`, function (t) { + const result = parseBuffer(buf, [ + ['key1', 1], + ['key2', 2], + ['key3', 4], + ['key4', 8], + ]); + t.same(result, { + key1: 98, + key2: 26229, + key3: 4285687144, + key4: 3824536674483896300 + }); + t.end(); +}); + +test(`parse little endian values for decreasing byte size`, function (t) { + const result = parseBuffer(buf, [ + ['key1', 8], + ['key2', 4], + ['key3', 2], + ['key4', 1], + ]); + t.same(result, { + key1: 12609923261529487000, + key2: 3617132800, + key3: 4987, + key4: 53 + }); + t.end(); +}); + +test(`parse little endian values with null keys due to small buffer`, function (t) { + const result = parseBuffer(buf, [ + ['key1', 8], + ['key2', 8], + ['key3', 8], + ['key4', 8], + ]); + t.same(result, { + key1: 12609923261529487000, + key2: null, + key3: null, + key4: null + }); + t.end(); +}); \ No newline at end of file diff --git a/test/parseCompressedDirectory.js b/test/parseCompressedDirectory.js new file mode 100644 index 00000000..2adecb7c --- /dev/null +++ b/test/parseCompressedDirectory.js @@ -0,0 +1,40 @@ +import { test } from 'tap'; +import fs from 'fs'; +import temp from 'temp'; +import dirdiff from 'dirdiff'; +import { Extract } from '../index.js'; + +/* +zipinfo testData/compressed-directory-entry/archive.zip | grep META-INF/ + +?rwxr-xr-x 2.0 unx 0 b- defN 17-Sep-09 20:43 META-INF/ +?rw------- 2.0 unx 244 b- defN 17-Sep-09 20:43 META-INF/container.xml +*/ +test("extract compressed archive w/ a compressed directory entry", function (t) { + const archive = './testData/compressed-directory-entry/archive.zip'; + + temp.mkdir('node-unzip-', function (err, dirPath) { + if (err) { + throw err; + } + const unzipExtractor = Extract({ path: dirPath }); + unzipExtractor.on('error', function(err) { + throw err; + }); + unzipExtractor.on('close', testExtractionResults); + + fs.createReadStream(archive).pipe(unzipExtractor); + + function testExtractionResults() { + dirdiff('./testData/compressed-directory-entry/inflated', dirPath, { + fileContents: true + }, function (err, diffs) { + if (err) { + throw err; + } + t.equal(diffs.length, 0, 'extracted directory contents'); + t.end(); + }); + } + }); +}); \ No newline at end of file diff --git a/test/parseContent.js b/test/parseContent.js new file mode 100644 index 00000000..2fede7ae --- /dev/null +++ b/test/parseContent.js @@ -0,0 +1,21 @@ +import { test } from 'tap'; +import fs from 'fs'; +import { Parse } from '../index.js'; + +test("get content of a single file entry out of a zip", function (t) { + const archive = './testData/compressed-standard/archive.zip'; + + fs.createReadStream(archive) + .pipe(Parse()) + .on('entry', function(entry) { + if (entry.path !== 'file.txt') + return entry.autodrain(); + + entry.buffer() + .then(function(str) { + const fileStr = fs.readFileSync('./testData/compressed-standard/inflated/file.txt', 'utf8'); + t.equal(str.toString(), fileStr); + t.end(); + }); + }); +}); \ No newline at end of file diff --git a/test/parseDateTime.js b/test/parseDateTime.js new file mode 100644 index 00000000..583e20af --- /dev/null +++ b/test/parseDateTime.js @@ -0,0 +1,27 @@ +import { test } from 'tap'; +import { Open, ParseOne } from '../index.js'; +import fs from 'fs'; + +const archive = './testData/compressed-standard/archive.zip'; + +test('parse datetime using Open', function (t) { + return Open.file(archive) + .then(function(d) { + const file = d.files.filter(function(file) { + return file.path == 'file.txt'; + })[0]; + t.same(file.lastModifiedDateTime, new Date('2012-08-08T11:21:10.000Z')); + t.end(); + }); +}); + +test('parse datetime using Parse', function(t) { + fs.createReadStream(archive) + .pipe(ParseOne('file.txt')) + .on('entry', function(entry) { + if (entry.path !== 'file.txt') + return entry.autodrain(); + t.same(entry.vars.lastModifiedDateTime, new Date('2012-08-08T11:21:10.000Z')); + t.end(); + }); +}); diff --git a/test/parseOneEntry.js b/test/parseOneEntry.js new file mode 100644 index 00000000..4ccd2573 --- /dev/null +++ b/test/parseOneEntry.js @@ -0,0 +1,50 @@ +'use strict'; + +import { test } from 'tap'; +import fs from 'fs'; +import streamBuffers from "stream-buffers"; +import { ParseOne } from '../index.js'; + +const archive = './testData/compressed-standard/archive.zip'; + +test("pipe a single file entry out of a zip", function (t) { + const writableStream = new streamBuffers.WritableStreamBuffer(); + writableStream.on('close', function () { + const str = writableStream.getContentsAsString('utf8'); + const fileStr = fs.readFileSync('./testData/compressed-standard/inflated/file.txt', 'utf8'); + t.equal(str, fileStr); + t.end(); + }); + + fs.createReadStream(archive) + .pipe(ParseOne('file.txt')) + .pipe(writableStream); +}); + +test('errors if file is not found', function (t) { + fs.createReadStream(archive) + .pipe(ParseOne('not_exists')) + .on('error', function(e) { + t.equal(e.message, 'PATTERN_NOT_FOUND'); + t.end(); + }); +}); + +test('error - invalid signature', function(t) { + ParseOne() + .on('error', function(e) { + t.equal(e.message.indexOf('invalid signature'), 0); + t.end(); + }) + .end('this is not a zip file'); +}); + +test('error - file ended', function(t) { + ParseOne() + .on('error', function(e) { + if (e.message == 'PATTERN_NOT_FOUND') return; + t.equal(e.message, 'FILE_ENDED'); + t.end(); + }) + .end('t'); +}); \ No newline at end of file diff --git a/test/parsePromise.js b/test/parsePromise.js new file mode 100644 index 00000000..99d5a29d --- /dev/null +++ b/test/parsePromise.js @@ -0,0 +1,49 @@ +'use strict'; + +import { test } from 'tap'; +import fs from 'fs'; +import { Parse } from '../index.js'; + +let entryRead; + +test("promise should resolve when entries have been processed", function (t) { + const archive = './testData/compressed-standard/archive.zip'; + + fs.createReadStream(archive) + .pipe(Parse()) + .on('entry', function(entry) { + if (entry.path !== 'file.txt') + return entry.autodrain(); + + entry.buffer() + .then(function() { + entryRead = true; + }); + }) + .promise() + .then(function() { + t.equal(entryRead, true); + t.end(); + }, function() { + t.fail('This project should resolve'); + t.end(); + }); +}); + +test("promise should be rejected if there is an error in the stream", function (t) { + const archive = './testData/compressed-standard/archive.zip'; + + fs.createReadStream(archive) + .pipe(Parse()) + .on('entry', function() { + this.emit('error', new Error('this is an error')); + }) + .promise() + .then(function() { + t.fail('This promise should be rejected'); + t.end(); + }, function(e) { + t.equal(e.message, 'this is an error'); + t.end(); + }); +}); \ No newline at end of file diff --git a/test/pipeSingleEntry.js b/test/pipeSingleEntry.js index d29b4eba..5316c92a 100644 --- a/test/pipeSingleEntry.js +++ b/test/pipeSingleEntry.js @@ -1,23 +1,19 @@ -'use strict'; - -var test = require('tap').test; -var fs = require('fs'); -var path = require('path'); -var temp = require('temp'); -var streamBuffers = require("stream-buffers"); -var unzip = require('../'); +import { test } from 'tap'; +import fs from 'fs'; +import streamBuffers from "stream-buffers"; +import { Parse } from '../index.js'; test("pipe a single file entry out of a zip", function (t) { - var archive = path.join(__dirname, '../testData/compressed-standard/archive.zip'); + const archive = './testData/compressed-standard/archive.zip'; fs.createReadStream(archive) - .pipe(unzip.Parse()) + .pipe(Parse()) .on('entry', function(entry) { if (entry.path === 'file.txt') { - var writableStream = new streamBuffers.WritableStreamBuffer(); + const writableStream = new streamBuffers.WritableStreamBuffer(); writableStream.on('close', function () { - var str = writableStream.getContentsAsString('utf8'); - var fileStr = fs.readFileSync(path.join(__dirname, '../testData/compressed-standard/inflated/file.txt'), 'utf8') + const str = writableStream.getContentsAsString('utf8'); + const fileStr = fs.readFileSync('./testData/compressed-standard/inflated/file.txt', 'utf8'); t.equal(str, fileStr); t.end(); }); diff --git a/test/streamSingleEntry.js b/test/streamSingleEntry.js new file mode 100644 index 00000000..d0ff462d --- /dev/null +++ b/test/streamSingleEntry.js @@ -0,0 +1,32 @@ +import { test } from 'tap'; +import fs from 'fs'; +import streamBuffers from "stream-buffers"; +import { Parse } from '../index.js'; +import Stream from 'stream'; + +test("pipe a single file entry out of a zip", function (t) { + const receiver = Stream.Transform({objectMode:true}); + receiver._transform = function(entry, e, cb) { + if (entry.path === 'file.txt') { + const writableStream = new streamBuffers.WritableStreamBuffer(); + writableStream.on('close', function () { + const str = writableStream.getContentsAsString('utf8'); + const fileStr = fs.readFileSync('./testData/compressed-standard/inflated/file.txt', 'utf8'); + t.equal(str, fileStr); + t.end(); + cb(); + }); + entry.pipe(writableStream); + } else { + entry.autodrain(); + cb(); + } + }; + + const archive = './testData/compressed-standard/archive.zip'; + + fs.createReadStream(archive) + .pipe(Parse()) + .pipe(receiver); + +}); diff --git a/test/uncompressed.js b/test/uncompressed.js index fcbf3beb..3159b108 100644 --- a/test/uncompressed.js +++ b/test/uncompressed.js @@ -1,16 +1,15 @@ -'use strict'; - -var test = require('tap').test; -var fs = require('fs'); -var path = require('path'); -var temp = require('temp'); -var dirdiff = require('dirdiff'); -var unzip = require('../'); +import { test } from 'tap'; +import fs from 'fs'; +import os from 'os'; +import path from 'path'; +import temp from 'temp'; +import dirdiff from 'dirdiff'; +import { Extract, Parse } from '../index.js'; test("parse uncompressed archive", function (t) { - var archive = path.join(__dirname, '../testData/uncompressed/archive.zip'); + const archive = './testData/uncompressed/archive.zip'; - var unzipParser = unzip.Parse(); + const unzipParser = Parse(); fs.createReadStream(archive).pipe(unzipParser); unzipParser.on('error', function(err) { throw err; @@ -20,13 +19,13 @@ test("parse uncompressed archive", function (t) { }); test("extract uncompressed archive", function (t) { - var archive = path.join(__dirname, '../testData/uncompressed/archive.zip'); + const archive = './testData/uncompressed/archive.zip'; temp.mkdir('node-unzip-', function (err, dirPath) { if (err) { throw err; } - var unzipExtractor = unzip.Extract({ path: dirPath }); + const unzipExtractor = Extract({ path: dirPath }); unzipExtractor.on('error', function(err) { throw err; }); @@ -35,7 +34,7 @@ test("extract uncompressed archive", function (t) { fs.createReadStream(archive).pipe(unzipExtractor); function testExtractionResults() { - dirdiff(path.join(__dirname, '../testData/uncompressed/inflated'), dirPath, { + dirdiff('./testData/uncompressed/inflated', dirPath, { fileContents: true }, function (err, diffs) { if (err) { @@ -46,4 +45,94 @@ test("extract uncompressed archive", function (t) { }); } }); -}); \ No newline at end of file +}); + +test("do not extract zip slip archive", function (t) { + const archive = './testData/zip-slip/zip-slip.zip'; + + temp.mkdir('node-zipslip-', function (err, dirPath) { + if (err) { + throw err; + } + const unzipExtractor = Extract({ path: dirPath }); + unzipExtractor.on('error', function(err) { + throw err; + }); + unzipExtractor.on('close', testNoSlip); + + fs.createReadStream(archive).pipe(unzipExtractor); + + function testNoSlip() { + const mode = fs.F_OK | (fs.constants && fs.constants.F_OK); + return fs.access(path.join(os.tmpdir(), 'evil.txt'), mode, evilFileCallback); + } + + function evilFileCallback(err) { + if (err) { + t.pass('no zip slip'); + } else { + t.fail('evil file created'); + } + return t.end(); + } + + }); +}); + +function testZipSlipArchive(t, slipFileName, attackPathFactory){ + const archive = path.join('./testData/zip-slip', slipFileName); + + temp.mkdir('node-zipslip-' + slipFileName, function (err, dirPath) { + if (err) { + throw err; + } + const attackPath = attackPathFactory(dirPath); + CheckForSlip(attackPath, function(slipAlreadyExists){ + if(slipAlreadyExists){ + t.fail('Cannot check for slip because the slipped file already exists at "' + attackPath+ '"'); + t.end(); + } + else{ + const unzipExtractor = Extract({ path: dirPath }); + unzipExtractor.on('error', function(err) { + throw err; + }); + unzipExtractor.on('close', testNoSlip); + + fs.createReadStream(archive).pipe(unzipExtractor); + } + }); + + function CheckForSlip(path, resultCallback) { + const fsCallback = function(err){ return resultCallback(!err); }; + const mode = fs.F_OK | (fs.constants && fs.constants.F_OK); + return fs.access(path, mode, fsCallback); + } + + function testNoSlip() { + CheckForSlip(attackPath, function(slipExists) { + if (slipExists) { + t.fail('evil file created from ' + slipFileName + ' at "' + attackPath + '"'); + fs.unlinkSync(attackPath); + } else { + t.pass('no zip slip from ' + slipFileName); + } + return t.end(); + }); + } + }); +} + +test("do not extract zip slip archive(Windows)", function (t) { + let pathFactory; + if(process.platform === "win32") { + pathFactory = function() { return '\\Temp\\evil.txt'; }; + } + else{ + // UNIX should treat the backslashes as escapes not directory delimiters + // will be a file with slashes in the name. Looks real weird. + pathFactory = function(dirPath) { return path.join(dirPath, '..\\..\\..\\..\\..\\..\\..\\..\\..\\..\\..\\..\\..\\..\\..\\..\\..\\..\\..\\..\\..\\..\\..\\..\\..\\..\\..\\..\\..\\..\\..\\..\\..\\..\\..\\..\\..\\..\\..\\..\\Temp\\evil.txt'); }; + } + + testZipSlipArchive(t, 'zip-slip-win.zip', pathFactory); +}); diff --git a/test/zip64.js b/test/zip64.js new file mode 100644 index 00000000..f8be1cff --- /dev/null +++ b/test/zip64.js @@ -0,0 +1,161 @@ +'use strict'; + +import t from 'tap'; +import { Parse, Extract, Open } from '../index.js'; +import fs from 'fs'; +import temp from 'temp'; + +const UNCOMPRESSED_SIZE = 5368709120; +const ZIP64_OFFSET = 72; +const ZIP64_SIZE = 36; +const ZIP64_EXTRA_FIELD_OFFSET = 0; + +t.test('Correct uncompressed size for zip64', function (t) { + const archive = './testData/big.zip'; + + t.test('in unzipper.Open', function(t) { + Open.file(archive) + .then(function(d) { + const file = d.files[0]; + t.same(file.uncompressedSize, UNCOMPRESSED_SIZE, 'Open: Directory header'); + + d.files[0].stream() + .on('vars', function(vars) { + t.same(vars.uncompressedSize, UNCOMPRESSED_SIZE, 'Open: File header'); + t.end(); + }) + .on('error', function(e) { + t.same(e.message, 'FILE_ENDED'); + t.end(); + }); + }); + }); + + t.test('in unzipper.parse', function(t) { + fs.createReadStream(archive) + .pipe(Parse()) + .on('entry', function(entry) { + t.same(entry.vars.uncompressedSize, UNCOMPRESSED_SIZE, 'Parse: File header'); + t.end(); + }); + }); + + t.end(); +}); + +t.test('Parse files with all zip64 extra fields correctly', function (t) { + const archive = './testData/zip64-allextrafields.zip'; + + t.test('in unzipper.Open', function(t) { + Open.file(archive) + .then(function(d) { + d.files[0].stream() + .on('vars', function(vars) { + t.same(vars.extra.uncompressedSize, ZIP64_SIZE, 'Open: Extra field'); + t.same(vars.extra.compressedSize, ZIP64_SIZE, 'Open: Extra field'); + t.same(vars.extra.offsetToLocalFileHeader, ZIP64_EXTRA_FIELD_OFFSET, 'Open: Extra field'); + t.same(vars.uncompressedSize, ZIP64_SIZE, 'Open: File header'); + t.same(vars.compressedSize, ZIP64_SIZE, 'Open: File header'); + t.same(vars.offsetToLocalFileHeader, ZIP64_EXTRA_FIELD_OFFSET, 'Open: File header'); + t.end(); + }) + .on('error', function(e) { + t.same(e.message, 'FILE_ENDED'); + t.end(); + }); + }); + }); + + t.test('in unzipper.extract', function (t) { + temp.mkdir('node-unzip-', function (err, dirPath) { + if (err) { + throw err; + } + fs.createReadStream(archive) + .pipe(Extract({ path: dirPath })) + .on('close', function() { t.end(); }); + }); + }); + + t.end(); +}); + +t.test('Parse files with zip64 extra field with only offset length correctly', function (t) { + const archive = './testData/zip64-extrafieldoffsetlength.zip'; + + t.test('in unzipper.Open', function(t) { + Open.file(archive) + .then(function(d) { + d.files[0].stream() + .on('vars', function(vars) { + t.same(vars.extra.offsetToLocalFileHeader, ZIP64_EXTRA_FIELD_OFFSET, 'Open: Extra field'); + t.same(vars.offsetToLocalFileHeader, ZIP64_EXTRA_FIELD_OFFSET, 'Open: File header'); + t.end(); + }) + .on('error', function(e) { + t.same(e.message, 'FILE_ENDED'); + t.end(); + }); + }); + }); + + t.test('in unzipper.extract', function (t) { + temp.mkdir('node-unzip-', function (err, dirPath) { + if (err) { + throw err; + } + fs.createReadStream(archive) + .pipe(Extract({ path: dirPath })) + .on('close', function() { t.end(); }); + }); + }); + + t.end(); +}); + +t.test('Parse files from regular zip64 format correctly', function (t) { + const archive = './testData/zip64.zip'; + + t.test('in unzipper.Open', function(t) { + Open.file(archive) + .then(function(d) { + t.same(d.offsetToStartOfCentralDirectory, ZIP64_OFFSET, 'Open: Directory header'); + t.same(d.files.length, 1, 'Open: Files Size'); + + d.files[0].stream() + .on('vars', function(vars) { + t.same(vars.offsetToLocalFileHeader, 0, 'Open: File header'); + t.same(vars.uncompressedSize, ZIP64_SIZE, 'Open: File header'); + t.same(vars.compressedSize, ZIP64_SIZE, 'Open: File header'); + t.same(vars.path, 'README', 'Open: File header'); + t.end(); + }) + .on('error', function(e) { + t.same(e.message, 'FILE_ENDED'); + t.end(); + }); + }); + }); + + t.test('in unzipper.parse', function(t) { + fs.createReadStream(archive) + .pipe(Parse()) + .on('entry', function(entry) { + t.same(entry.vars.uncompressedSize, ZIP64_SIZE, 'Parse: File header'); + }) + .on('close', function() { t.end(); }); + }); + + t.test('in unzipper.extract', function (t) { + temp.mkdir('node-unzip-', function (err, dirPath) { + if (err) { + throw err; + } + fs.createReadStream(archive) + .pipe(Extract({ path: dirPath })) + .on('close', function() { t.end(); }); + }); + }); + + t.end(); +}); diff --git a/test/zipSlipSiblingPrefix.js b/test/zipSlipSiblingPrefix.js new file mode 100644 index 00000000..96c62e6a --- /dev/null +++ b/test/zipSlipSiblingPrefix.js @@ -0,0 +1,163 @@ +import { test } from 'tap'; +import fs from 'fs'; +import path from 'path'; +import os from 'os'; +import { Readable } from 'stream'; // 'node:stream' +import { Open, Extract } from '../index.js'; + +// Compute CRC32 for STORED zip entries +function crc32(buf) { + if (!crc32._table) { + const t = crc32._table = new Uint32Array(256); + for (let i = 0; i < 256; i++) { + let c = i; + for (let j = 0; j < 8; j++) c = (c & 1) ? (0xEDB88320 ^ (c >>> 1)) : (c >>> 1); + t[i] = c; + } + } + let crc = 0xFFFFFFFF; + for (let i = 0; i < buf.length; i++) crc = crc32._table[(crc ^ buf[i]) & 0xFF] ^ (crc >>> 8); + return (crc ^ 0xFFFFFFFF) >>> 0; +} + +// Build a minimal STORED zip containing a single entry +function makeZip(entryPath, content) { + const data = Buffer.from(content); + const name = Buffer.from(entryPath); + const crc = crc32(data); + + const local = Buffer.alloc(30 + name.length + data.length); + local.writeUInt32LE(0x04034b50, 0); + local.writeUInt16LE(20, 4); + local.writeUInt16LE(0, 6); + local.writeUInt16LE(0, 8); + local.writeUInt16LE(0, 10); + local.writeUInt16LE(0, 12); + local.writeUInt32LE(crc, 14); + local.writeUInt32LE(data.length, 18); + local.writeUInt32LE(data.length, 22); + local.writeUInt16LE(name.length, 26); + local.writeUInt16LE(0, 28); + name.copy(local, 30); + data.copy(local, 30 + name.length); + + const cdOffset = local.length; + const cd = Buffer.alloc(46 + name.length); + cd.writeUInt32LE(0x02014b50, 0); + cd.writeUInt16LE(20, 4); + cd.writeUInt16LE(20, 6); + cd.writeUInt16LE(0, 8); + cd.writeUInt16LE(0, 10); + cd.writeUInt16LE(0, 12); + cd.writeUInt16LE(0, 14); + cd.writeUInt32LE(crc, 16); + cd.writeUInt32LE(data.length, 20); + cd.writeUInt32LE(data.length, 24); + cd.writeUInt16LE(name.length, 28); + cd.writeUInt16LE(0, 30); + cd.writeUInt16LE(0, 32); + cd.writeUInt16LE(0, 34); + cd.writeUInt16LE(0, 36); + cd.writeUInt32LE(0, 38); + cd.writeUInt32LE(0, 42); + name.copy(cd, 46); + + const eocd = Buffer.alloc(22); + eocd.writeUInt32LE(0x06054b50, 0); + eocd.writeUInt16LE(0, 4); + eocd.writeUInt16LE(0, 6); + eocd.writeUInt16LE(1, 8); + eocd.writeUInt16LE(1, 10); + eocd.writeUInt32LE(cd.length, 12); + eocd.writeUInt32LE(cdOffset, 16); + eocd.writeUInt16LE(0, 20); + + return Buffer.concat([local, cd, eocd]); +} + +let _seq = 0; +function makeTempDir() { + const dir = path.join(os.tmpdir(), 'unzipper-zipslip-' + process.pid + '-' + (++_seq)); + fs.mkdirSync(dir, { recursive: true }); + return dir; +} + +test('Extract blocks sibling-prefix path traversal via unzipper.Extract', function(t) { + const tmpDir = makeTempDir(); + const dest = path.join(tmpDir, 'dest'); + const sibling = path.join(tmpDir, 'dest-evil'); + fs.mkdirSync(dest); + + const zip = makeZip('../dest-evil/escaped.txt', 'pwned'); + const src = new Readable({ read() {} }); + src.push(zip); + src.push(null); + + const extractor = src.pipe(Extract({ path: dest })); + extractor.on('close', function() { + t.notOk(fs.existsSync(path.join(sibling, 'escaped.txt')), 'file must not escape to sibling directory'); + t.end(); + }); + extractor.on('error', function() { + t.notOk(fs.existsSync(path.join(sibling, 'escaped.txt')), 'file must not escape to sibling directory'); + t.end(); + }); +}); + +test('Extract allows normal entries within destination', function(t) { + const tmpDir = makeTempDir(); + const dest = path.join(tmpDir, 'dest'); + fs.mkdirSync(dest); + + const zip = makeZip('subdir/file.txt', 'hello'); + const src = new Readable({ read() {} }); + src.push(zip); + src.push(null); + + const extractor = src.pipe(Extract({ path: dest })); + extractor.on('close', function() { + t.ok(fs.existsSync(path.join(dest, 'subdir', 'file.txt')), 'normal entry must be extracted'); + t.end(); + }); + extractor.on('error', t.fail.bind(t)); +}); + +test('Open.extract blocks sibling-prefix path traversal', function(t) { + const tmpDir = makeTempDir(); + const dest = path.join(tmpDir, 'dest'); + const sibling = path.join(tmpDir, 'dest-evil'); + fs.mkdirSync(dest); + + const zip = makeZip('../dest-evil/escaped.txt', 'pwned'); + + Open.buffer(zip) + .then(function(d) { + return d.extract({ path: dest }); + }) + .then(function() { + t.notOk(fs.existsSync(path.join(sibling, 'escaped.txt')), 'file must not escape to sibling directory'); + t.end(); + }) + .catch(function() { + t.notOk(fs.existsSync(path.join(sibling, 'escaped.txt')), 'file must not escape to sibling directory'); + t.end(); + }); +}); + +test('Open.extract allows normal entries within destination', function(t) { + const tmpDir = makeTempDir(); + const dest = path.join(tmpDir, 'dest'); + fs.mkdirSync(dest); + + const zip = makeZip('subdir/file.txt', 'hello'); + + Open.buffer(zip) + .then(function(d) { + return d.extract({ path: dest }); + }) + .then(function() { + t.ok(fs.existsSync(path.join(dest, 'subdir', 'file.txt')), 'normal entry must be extracted'); + t.end(); + }) + .catch(t.fail.bind(t)); +}); diff --git a/testData/big.zip b/testData/big.zip new file mode 100644 index 00000000..57c1635f Binary files /dev/null and b/testData/big.zip differ diff --git a/testData/bootstrap-reboot.min.css b/testData/bootstrap-reboot.min.css new file mode 100644 index 00000000..ced04682 --- /dev/null +++ b/testData/bootstrap-reboot.min.css @@ -0,0 +1,8 @@ +/*! + * Bootstrap Reboot v4.0.0 (https://getbootstrap.com) + * Copyright 2011-2018 The Bootstrap Authors + * Copyright 2011-2018 Twitter, Inc. + * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE) + * Forked from Normalize.css, licensed MIT (https://github.com/necolas/normalize.css/blob/master/LICENSE.md) + */*,::after,::before{box-sizing:border-box}html{font-family:sans-serif;line-height:1.15;-webkit-text-size-adjust:100%;-ms-text-size-adjust:100%;-ms-overflow-style:scrollbar;-webkit-tap-highlight-color:transparent}@-ms-viewport{width:device-width}article,aside,dialog,figcaption,figure,footer,header,hgroup,main,nav,section{display:block}body{margin:0;font-family:-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,"Helvetica Neue",Arial,sans-serif,"Apple Color Emoji","Segoe UI Emoji","Segoe UI Symbol";font-size:1rem;font-weight:400;line-height:1.5;color:#212529;text-align:left;background-color:#fff}[tabindex="-1"]:focus{outline:0!important}hr{box-sizing:content-box;height:0;overflow:visible}h1,h2,h3,h4,h5,h6{margin-top:0;margin-bottom:.5rem}p{margin-top:0;margin-bottom:1rem}abbr[data-original-title],abbr[title]{text-decoration:underline;-webkit-text-decoration:underline dotted;text-decoration:underline dotted;cursor:help;border-bottom:0}address{margin-bottom:1rem;font-style:normal;line-height:inherit}dl,ol,ul{margin-top:0;margin-bottom:1rem}ol ol,ol ul,ul ol,ul ul{margin-bottom:0}dt{font-weight:700}dd{margin-bottom:.5rem;margin-left:0}blockquote{margin:0 0 1rem}dfn{font-style:italic}b,strong{font-weight:bolder}small{font-size:80%}sub,sup{position:relative;font-size:75%;line-height:0;vertical-align:baseline}sub{bottom:-.25em}sup{top:-.5em}a{color:#007bff;text-decoration:none;background-color:transparent;-webkit-text-decoration-skip:objects}a:hover{color:#0056b3;text-decoration:underline}a:not([href]):not([tabindex]){color:inherit;text-decoration:none}a:not([href]):not([tabindex]):focus,a:not([href]):not([tabindex]):hover{color:inherit;text-decoration:none}a:not([href]):not([tabindex]):focus{outline:0}code,kbd,pre,samp{font-family:monospace,monospace;font-size:1em}pre{margin-top:0;margin-bottom:1rem;overflow:auto;-ms-overflow-style:scrollbar}figure{margin:0 0 1rem}img{vertical-align:middle;border-style:none}svg:not(:root){overflow:hidden}table{border-collapse:collapse}caption{padding-top:.75rem;padding-bottom:.75rem;color:#6c757d;text-align:left;caption-side:bottom}th{text-align:inherit}label{display:inline-block;margin-bottom:.5rem}button{border-radius:0}button:focus{outline:1px dotted;outline:5px auto -webkit-focus-ring-color}button,input,optgroup,select,textarea{margin:0;font-family:inherit;font-size:inherit;line-height:inherit}button,input{overflow:visible}button,select{text-transform:none}[type=reset],[type=submit],button,html [type=button]{-webkit-appearance:button}[type=button]::-moz-focus-inner,[type=reset]::-moz-focus-inner,[type=submit]::-moz-focus-inner,button::-moz-focus-inner{padding:0;border-style:none}input[type=checkbox],input[type=radio]{box-sizing:border-box;padding:0}input[type=date],input[type=datetime-local],input[type=month],input[type=time]{-webkit-appearance:listbox}textarea{overflow:auto;resize:vertical}fieldset{min-width:0;padding:0;margin:0;border:0}legend{display:block;width:100%;max-width:100%;padding:0;margin-bottom:.5rem;font-size:1.5rem;line-height:inherit;color:inherit;white-space:normal}progress{vertical-align:baseline}[type=number]::-webkit-inner-spin-button,[type=number]::-webkit-outer-spin-button{height:auto}[type=search]{outline-offset:-2px;-webkit-appearance:none}[type=search]::-webkit-search-cancel-button,[type=search]::-webkit-search-decoration{-webkit-appearance:none}::-webkit-file-upload-button{font:inherit;-webkit-appearance:button}output{display:inline-block}summary{display:list-item;cursor:pointer}template{display:none}[hidden]{display:none!important} +/*# sourceMappingURL=bootstrap-reboot.min.css.map */ \ No newline at end of file diff --git a/testData/chunk-boundary/chunk-boundary-archive.zip b/testData/chunk-boundary/chunk-boundary-archive.zip new file mode 100644 index 00000000..6236412e Binary files /dev/null and b/testData/chunk-boundary/chunk-boundary-archive.zip differ diff --git a/testData/chunk-boundary/chunk-boundary-archive/chunk-boundary.txt b/testData/chunk-boundary/chunk-boundary-archive/chunk-boundary.txt new file mode 100644 index 00000000..088518e6 --- /dev/null +++ b/testData/chunk-boundary/chunk-boundary-archive/chunk-boundary.txt @@ -0,0 +1,21 @@ +NOTE: This file is the right size so that it falls on a chunk boundary that triggers a certain edge case. + +P4qartlYyqSDttnVpe4v +BD3VC7hAkKvHsdtYLK14 +5TcfIgRymcFSwvnO3V46 +bhLvgitwkxZ7cWWWhOYP +5XtKpWsiQ49TTxCTDk37 +ouJaZcjJ6tXapepjRfmX +2jUSvc6XgzwK4usjUsbT +54aomOUZNzlCayHsDb79 +2JPKY7kmSAZ0nRFmyNn7 +kWOG6uKN3SAGyTVLvH2e +EodGFnPzNouhD82bOQvd +iWiDd0bCCXMU9UTJBlrN +WkuVIbdsd21fD6KLPj76 +W8BCTE5fZtIaU8PJZQT4 +VEtlz8NQBZGjDgWnIXCC +PlfuXRqMp60U7j2ct0H8 +ltm1tpAOEUM53Cn0iYQv +SKEx5n5kn4KqXaZYmL1K +a \ No newline at end of file diff --git a/testData/chunk-boundary/chunk-boundary-archive/chunk-boundary2.txt b/testData/chunk-boundary/chunk-boundary-archive/chunk-boundary2.txt new file mode 100644 index 00000000..aa1f3707 --- /dev/null +++ b/testData/chunk-boundary/chunk-boundary-archive/chunk-boundary2.txt @@ -0,0 +1,24 @@ +NOTE: This file is the right size so that it falls on a chunk boundary that triggers a certain edge case. + +P4qartlYyqSDttnVpe4v +BD3VC7hAkKvHsdtYLK14 +5TcfIgRymcFSwvnO3V46 +bhLvgitwkxZ7cWWWhOYP +5XtKpWsiQ49TTxCTDk37 +ouJaZcjJ6tXapepjRfmX +2jUSvc6XgzwK4usjUsbT +54aomOUZNzlCayHsDb79 +2JPKY7kmSAZ0nRFmyNn7 +kWOG6uKN3SAGyTVLvH2e +EodGFnPzNouhD82bOQvd +iWiDd0bCCXMU9UTJBlrN +WkuVIbdsd21fD6KLPj76 +W8BCTE5fZtIaU8PJZQT4 +VEtlz8NQBZGjDgWnIXCC +PlfuXRqMp60U7j2ct0H8 +ltm1tpAOEUM53Cn0iYQv +SKEx5n5kn4KqXaZYmL1K +RMuaEfuum3KGzkDl3pzc +zLcrrMiU7iGg7zV1HHPY +9xx1YEEFRyKa43esq2bd +fdSn6bQ4Vjc9bhylHB27 \ No newline at end of file diff --git a/testData/compressed-comment/archive.zip b/testData/compressed-comment/archive.zip new file mode 100644 index 00000000..14a862f1 Binary files /dev/null and b/testData/compressed-comment/archive.zip differ diff --git a/testData/compressed-comment/inflated/dir/fileInsideDir.txt b/testData/compressed-comment/inflated/dir/fileInsideDir.txt new file mode 100644 index 00000000..d81cc071 --- /dev/null +++ b/testData/compressed-comment/inflated/dir/fileInsideDir.txt @@ -0,0 +1 @@ +42 diff --git a/testData/compressed-comment/inflated/file.txt b/testData/compressed-comment/inflated/file.txt new file mode 100644 index 00000000..210e1e1b --- /dev/null +++ b/testData/compressed-comment/inflated/file.txt @@ -0,0 +1 @@ +node.js rocks diff --git a/testData/compressed-cp866/archive.zip b/testData/compressed-cp866/archive.zip new file mode 100644 index 00000000..04bd3c93 Binary files /dev/null and b/testData/compressed-cp866/archive.zip differ diff --git "a/testData/compressed-cp866/inflated/\320\242\320\265\321\201\321\202.txt" "b/testData/compressed-cp866/inflated/\320\242\320\265\321\201\321\202.txt" new file mode 100644 index 00000000..2f29f70d --- /dev/null +++ "b/testData/compressed-cp866/inflated/\320\242\320\265\321\201\321\202.txt" @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/testData/compressed-directory-entry/archive.zip b/testData/compressed-directory-entry/archive.zip new file mode 100644 index 00000000..e81a6aa7 Binary files /dev/null and b/testData/compressed-directory-entry/archive.zip differ diff --git a/testData/compressed-directory-entry/inflated/META-INF/container.xml b/testData/compressed-directory-entry/inflated/META-INF/container.xml new file mode 100644 index 00000000..f17cad9a --- /dev/null +++ b/testData/compressed-directory-entry/inflated/META-INF/container.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/testData/compressed-directory-entry/inflated/content.opf b/testData/compressed-directory-entry/inflated/content.opf new file mode 100644 index 00000000..d3ff6366 --- /dev/null +++ b/testData/compressed-directory-entry/inflated/content.opf @@ -0,0 +1,22 @@ + + + + calibre (3.15.0) [https://calibre-ebook.com] + 2006-03-06T20:06:33+00:00 + 8ee1add8-e31f-4b26-8059-e939a3190706 + en + Author text + Title text + + + + + + + + + + + + + diff --git a/testData/compressed-directory-entry/inflated/index.html b/testData/compressed-directory-entry/inflated/index.html new file mode 100644 index 00000000..7ddea554 --- /dev/null +++ b/testData/compressed-directory-entry/inflated/index.html @@ -0,0 +1,15 @@ + + + + Blank PDF Document + + + + + + + + +

+ + diff --git a/testData/compressed-directory-entry/inflated/mimetype b/testData/compressed-directory-entry/inflated/mimetype new file mode 100644 index 00000000..57ef03f2 --- /dev/null +++ b/testData/compressed-directory-entry/inflated/mimetype @@ -0,0 +1 @@ +application/epub+zip \ No newline at end of file diff --git a/testData/compressed-directory-entry/inflated/page_styles.css b/testData/compressed-directory-entry/inflated/page_styles.css new file mode 100644 index 00000000..7ee6c2c8 --- /dev/null +++ b/testData/compressed-directory-entry/inflated/page_styles.css @@ -0,0 +1,4 @@ +@page { + margin-bottom: 5pt; + margin-top: 5pt + } diff --git a/testData/compressed-directory-entry/inflated/stylesheet.css b/testData/compressed-directory-entry/inflated/stylesheet.css new file mode 100644 index 00000000..749a0321 --- /dev/null +++ b/testData/compressed-directory-entry/inflated/stylesheet.css @@ -0,0 +1,11 @@ +.calibre { + display: block; + font-size: 1em; + padding-left: 0; + padding-right: 0; + margin: 0 5pt + } +.calibre1 { + display: block; + margin: 1em 0 + } diff --git a/testData/compressed-directory-entry/inflated/toc.ncx b/testData/compressed-directory-entry/inflated/toc.ncx new file mode 100644 index 00000000..80db2a6c --- /dev/null +++ b/testData/compressed-directory-entry/inflated/toc.ncx @@ -0,0 +1,21 @@ + + + + + + + + + + + Title text + + + + + Start + + + + + diff --git a/testData/compressed-encrypted/archive.zip b/testData/compressed-encrypted/archive.zip new file mode 100644 index 00000000..fd58fb4f Binary files /dev/null and b/testData/compressed-encrypted/archive.zip differ diff --git a/testData/compressed-encrypted/inflated/dir/fileInsideDir.txt b/testData/compressed-encrypted/inflated/dir/fileInsideDir.txt new file mode 100644 index 00000000..d81cc071 --- /dev/null +++ b/testData/compressed-encrypted/inflated/dir/fileInsideDir.txt @@ -0,0 +1 @@ +42 diff --git a/testData/compressed-encrypted/inflated/file.txt b/testData/compressed-encrypted/inflated/file.txt new file mode 100644 index 00000000..ac652242 --- /dev/null +++ b/testData/compressed-encrypted/inflated/file.txt @@ -0,0 +1,11 @@ +node.js rocks + +Lorem ipsum dolor sit amet, consectetur adipiscing elit. Cras commodo molestie nunc, eu pharetra libero accumsan nec. Vestibulum hendrerit, augue ac congue varius, enim metus congue quam, imperdiet gravida diam felis nec dui. Morbi ipsum enim, tristique nec congue a, commodo ac sapien. Praesent semper metus quis diam hendrerit ut condimentum eros lobortis. Aenean faucibus arcu nec leo aliquam tincidunt. Nunc bibendum dictum bibendum. Nunc ultricies pretium lacus, sit amet lobortis quam egestas quis. Fusce viverra magna rhoncus sem posuere non tempus nulla vestibulum. + +Sed aliquet, odio vel condimentum pellentesque, mauris risus iaculis elit, at congue erat mi at ante. In at dictum metus. Ut rutrum mauris felis. Nulla sed risus nunc, eget ultrices est. Nullam gravida diam in arcu vulputate varius. Sed id egestas magna. Ut a libero sapien. + +Integer congue felis ut nisl fringilla ac interdum est pretium. Proin tellus augue, molestie id ultricies placerat, ornare a felis. In eu nibh velit. Pellentesque cursus ultricies fermentum. Mauris eget velit tempor nulla bibendum accumsan sit amet a ante. Morbi rutrum tempor varius. Aenean congue leo vitae mi suscipit ac tempor nibh pulvinar. Maecenas risus eros, sodales quis tincidunt non, vulputate eget orci. Maecenas condimentum lectus pretium orci adipiscing interdum. Sed interdum vehicula urna ut scelerisque. + +Phasellus pellentesque tellus in neque auctor pellentesque adipiscing justo consequat. In tincidunt rhoncus mollis. Suspendisse quis est elit, vel semper lorem. Donec cursus, leo ac fermentum luctus, dui dolor pretium nunc, vel congue eros arcu sit amet enim. Nam nibh orci, laoreet id volutpat eu, aliquet sed ligula. Donec placerat sagittis leo, eget hendrerit nisi varius sed. In pharetra erat non justo interdum id tempus purus tempor. Cum sociis natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Suspendisse felis leo, pellentesque tristique iaculis consequat, vestibulum a erat. Curabitur ligula risus, consectetur at adipiscing sit amet, accumsan non justo. Proin ultricies molestie lorem et auctor. Duis commodo varius semper. Ut tempus porttitor dolor nec mattis. Cras massa eros, tincidunt eget placerat a, luctus eu arcu. Nulla ac orci vitae odio dapibus dictum vitae porta erat. + +Duis luctus convallis euismod. Integer orci massa, bibendum eu blandit quis, facilisis lobortis purus. Donec et sapien quis elit fermentum cursus a ut lacus. Nullam tellus felis, congue et pulvinar sit amet, luctus ac augue. Sed massa nunc, dignissim non viverra ac, dictum sit amet erat. Sed nunc tortor, convallis et tristique ut, aliquam ut orci. Integer nec magna vitae elit sagittis accumsan id ac mi. diff --git a/testData/compressed-standard-crx/archive.crx b/testData/compressed-standard-crx/archive.crx new file mode 100644 index 00000000..c07b21c6 Binary files /dev/null and b/testData/compressed-standard-crx/archive.crx differ diff --git a/testData/compressed-standard-crx/crxmake.sh b/testData/compressed-standard-crx/crxmake.sh new file mode 100755 index 00000000..0112d064 --- /dev/null +++ b/testData/compressed-standard-crx/crxmake.sh @@ -0,0 +1,31 @@ +#!/bin/bash -e +# +# Purpose: Pack a Chromium extension directory into crx format + +crx="archive.crx" +pub="publickey.tmp" +sig="signature.tmp" +zip="../compressed-standard/archive.zip" +key="./key" + +# signature +openssl sha1 -sha1 -binary -sign key.pem < "$zip" > "$sig" + +# public key +openssl rsa -pubout -outform DER < key.pem > "$pub" 2>/dev/null + +byte_swap () { + # Take "abcdefgh" and return it as "ghefcdab" + echo "${1:6:2}${1:4:2}${1:2:2}${1:0:2}" +} + +crmagic_hex="4372 3234" # Cr24 +version_hex="0200 0000" # 2 +pub_len_hex=$(byte_swap $(printf '%08x\n' $(ls -l "$pub" | awk '{print $5}'))) +sig_len_hex=$(byte_swap $(printf '%08x\n' $(ls -l "$sig" | awk '{print $5}'))) +( + echo "$crmagic_hex $version_hex $pub_len_hex $sig_len_hex" | xxd -r -p + cat "$pub" "$sig" "$zip" +) > "$crx" +rm *.tmp +echo "Wrote $crx" diff --git a/testData/compressed-standard-crx/inflated/dir/fileInsideDir.txt b/testData/compressed-standard-crx/inflated/dir/fileInsideDir.txt new file mode 100644 index 00000000..d81cc071 --- /dev/null +++ b/testData/compressed-standard-crx/inflated/dir/fileInsideDir.txt @@ -0,0 +1 @@ +42 diff --git a/testData/compressed-standard-crx/inflated/file.txt b/testData/compressed-standard-crx/inflated/file.txt new file mode 100644 index 00000000..ac652242 --- /dev/null +++ b/testData/compressed-standard-crx/inflated/file.txt @@ -0,0 +1,11 @@ +node.js rocks + +Lorem ipsum dolor sit amet, consectetur adipiscing elit. Cras commodo molestie nunc, eu pharetra libero accumsan nec. Vestibulum hendrerit, augue ac congue varius, enim metus congue quam, imperdiet gravida diam felis nec dui. Morbi ipsum enim, tristique nec congue a, commodo ac sapien. Praesent semper metus quis diam hendrerit ut condimentum eros lobortis. Aenean faucibus arcu nec leo aliquam tincidunt. Nunc bibendum dictum bibendum. Nunc ultricies pretium lacus, sit amet lobortis quam egestas quis. Fusce viverra magna rhoncus sem posuere non tempus nulla vestibulum. + +Sed aliquet, odio vel condimentum pellentesque, mauris risus iaculis elit, at congue erat mi at ante. In at dictum metus. Ut rutrum mauris felis. Nulla sed risus nunc, eget ultrices est. Nullam gravida diam in arcu vulputate varius. Sed id egestas magna. Ut a libero sapien. + +Integer congue felis ut nisl fringilla ac interdum est pretium. Proin tellus augue, molestie id ultricies placerat, ornare a felis. In eu nibh velit. Pellentesque cursus ultricies fermentum. Mauris eget velit tempor nulla bibendum accumsan sit amet a ante. Morbi rutrum tempor varius. Aenean congue leo vitae mi suscipit ac tempor nibh pulvinar. Maecenas risus eros, sodales quis tincidunt non, vulputate eget orci. Maecenas condimentum lectus pretium orci adipiscing interdum. Sed interdum vehicula urna ut scelerisque. + +Phasellus pellentesque tellus in neque auctor pellentesque adipiscing justo consequat. In tincidunt rhoncus mollis. Suspendisse quis est elit, vel semper lorem. Donec cursus, leo ac fermentum luctus, dui dolor pretium nunc, vel congue eros arcu sit amet enim. Nam nibh orci, laoreet id volutpat eu, aliquet sed ligula. Donec placerat sagittis leo, eget hendrerit nisi varius sed. In pharetra erat non justo interdum id tempus purus tempor. Cum sociis natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Suspendisse felis leo, pellentesque tristique iaculis consequat, vestibulum a erat. Curabitur ligula risus, consectetur at adipiscing sit amet, accumsan non justo. Proin ultricies molestie lorem et auctor. Duis commodo varius semper. Ut tempus porttitor dolor nec mattis. Cras massa eros, tincidunt eget placerat a, luctus eu arcu. Nulla ac orci vitae odio dapibus dictum vitae porta erat. + +Duis luctus convallis euismod. Integer orci massa, bibendum eu blandit quis, facilisis lobortis purus. Donec et sapien quis elit fermentum cursus a ut lacus. Nullam tellus felis, congue et pulvinar sit amet, luctus ac augue. Sed massa nunc, dignissim non viverra ac, dictum sit amet erat. Sed nunc tortor, convallis et tristique ut, aliquam ut orci. Integer nec magna vitae elit sagittis accumsan id ac mi. diff --git a/testData/compressed-standard-crx/key.pem b/testData/compressed-standard-crx/key.pem new file mode 100644 index 00000000..c23c6128 --- /dev/null +++ b/testData/compressed-standard-crx/key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDP/wGpId0UmDp9 +hLSepNEErUJGeATwDWGbNyGkK621CHPAMrDqNO8Z2XAimic6bbxUg6NxYLKwkxDM ++5kGY3lHFQkqmwO1nYwMTq8hSjmEtgfF+vF3EM5DJamv9vHVgQ6uW3v6VSvlSl7v +RlO8IgoIh1hTpVZN6w3rKIcQdENlTtfZJZIHWcv2oYp05+tw+hAWKB8WAmLrRzKF +tP9VrIPYIVClqhyOQR386d1U+8YW26LKOTQmLZ9Fy45bzQSYKA3qDObYustvncvR +1IOvTHKM+/2DPT+k9QvvWe0osKkDfG9lmsxO3Pv0XJ5F1TEjW621OychV46u4k2E +ugcqLw7JAgMBAAECggEBAK8kSrCxjCj4WmAxK6penIa0ohcWnwnIKsV5cgF8qiPD +fzx7Ms+0TRXWK39nkNq5Zpk/05P32d+npsEEpwFjJ888HmDa8Q+wHyqJ4xxEFRTz +9q22NjeNVjHid/VtGowuxT76V/YeW/0SV8hhzsafXxV5al3c3kR0Rl8a3Eh1rf6S +FZdP2tts/+jyX1A81o2+duTguuftyQPgTFv7ugsiEW3De/dwn53Xr3tELZZ41ilJ +h9dryoq3gqWG9d7EgQ5F3YOdkn/YJopYzIiCNprqSMOG056VW+Fx7gkj6uGiIjRr +UbeUy6fWbmXYZgn4UHCVBXbHsM4XFtDmyXwjZ7EQm9UCgYEA7Xj3/BkGGyNW7XBO +pFGI0+lIfp9sFUsqZcVEapkdwSApL+/M/jmV1PTBn3KMge3vf8AgGKhD4ttK/Z+i +6H2wlazOuZjRcRDdfO/4lOUQxd/FTnr2JDXsSYdKQoaM4p+HH1FRE12j/xVrPBBT +M29mQOHzZYEFFQ21AMLv+KLa99cCgYEA4DlOaigXeJekg8w/sjlYmWz7FyECZT09 +xIaLndt8JrY4hH1gk5issHsBSzpMgiPizX9hqk3PxSkGfpH3iW9cE+WNdKs6/fEI +tWpVVCZ3DfEy/r3KVgZDvJbEKuu1alBuNlfF3mKHQnHGj+VwIRNLzB6PcZ/EF67a +uZF/l98g2l8CgYEA2nTbJIXcsSBseldDcTQ0fEVx1FJSOrCAG0lC7BFZZu1wFlIy +sXhGFrbmXAkjqu840LvsiuJYORxlOzYcxmXCCZ8EOYaUvb+3EZUsh8TGDlIRj2Xc +g2k7qlSUAukGOABrbGsA+6C8GhAZKxMVhw6m8W8q2qi7BSgr57xsx70BVNcCgYAg +H3Whdb7vEuKJ00Ao21hbGqbaSGtcb6qithfYdLJTpXVxXbjxTEUpP2YPDfoaBuQe +RqqKSH2EpHz+sxDAisipPRDH7yQTb22s99/jn2MdBzokDrKnIlyf7wWJlJ037u/r +LyX01y7DkSM+SEOJKYeJZbNtNtNUBUPmo/agnmHJhwKBgQCwZ1m+X2vBRwJiopIz +nwd8lActXaw8uSl+ydEj+4dx8pV9I2NhahB6FxrMY9iGz0q6/vPyYvY4bZn1QLQ2 +FZtwNLwVpItalag3YTBAcF8tsQEDWJCjbpuTlQLXwXIQxUoSRC/rK95iaKkZX62+ +JnP43yc9/htpBX/RdPvMRD8Lpg== +-----END PRIVATE KEY----- diff --git a/testData/compressed-standard/archive.zip b/testData/compressed-standard/archive.zip index 327aab67..961bad25 100644 Binary files a/testData/compressed-standard/archive.zip and b/testData/compressed-standard/archive.zip differ diff --git a/testData/compressed-standard/broken.zip b/testData/compressed-standard/broken.zip new file mode 100644 index 00000000..d13ba6b7 Binary files /dev/null and b/testData/compressed-standard/broken.zip differ diff --git a/testData/compressed-standard/file.txt b/testData/compressed-standard/file.txt new file mode 100644 index 00000000..038d718d --- /dev/null +++ b/testData/compressed-standard/file.txt @@ -0,0 +1 @@ +testing diff --git a/testData/office/testfile.docx b/testData/office/testfile.docx new file mode 100644 index 00000000..3c764199 Binary files /dev/null and b/testData/office/testfile.docx differ diff --git a/testData/office/testfile.xlsx b/testData/office/testfile.xlsx new file mode 100644 index 00000000..591f700e Binary files /dev/null and b/testData/office/testfile.xlsx differ diff --git a/testData/zip-slip/zip-slip-win.zip b/testData/zip-slip/zip-slip-win.zip new file mode 100644 index 00000000..3474c88b Binary files /dev/null and b/testData/zip-slip/zip-slip-win.zip differ diff --git a/testData/zip-slip/zip-slip.zip b/testData/zip-slip/zip-slip.zip new file mode 100644 index 00000000..38b3f499 Binary files /dev/null and b/testData/zip-slip/zip-slip.zip differ diff --git a/testData/zip64-allextrafields.zip b/testData/zip64-allextrafields.zip new file mode 100644 index 00000000..34f6884b Binary files /dev/null and b/testData/zip64-allextrafields.zip differ diff --git a/testData/zip64-extrafieldoffsetlength.zip b/testData/zip64-extrafieldoffsetlength.zip new file mode 100644 index 00000000..eeb410e0 Binary files /dev/null and b/testData/zip64-extrafieldoffsetlength.zip differ diff --git a/testData/zip64.zip b/testData/zip64.zip new file mode 100644 index 00000000..a2ee1fa3 Binary files /dev/null and b/testData/zip64.zip differ diff --git a/unzip.js b/unzip.js deleted file mode 100644 index f58bf069..00000000 --- a/unzip.js +++ /dev/null @@ -1,4 +0,0 @@ -'use strict'; - -exports.Parse = require('./lib/parse'); -exports.Extract = require('./lib/extract'); \ No newline at end of file