From 3c3eb9ce2839e62b844acc8cebaac984af329deb Mon Sep 17 00:00:00 2001 From: erseco Date: Sat, 4 Jul 2026 10:58:35 +0100 Subject: [PATCH 1/2] Fix silent corruption for backreference offsets beyond 2^25 The 4-byte offset bit-read in the sequence loop supplies only 32 - (spos & 7) usable bits, so offset codes >= 26 could silently lose their high bits whenever the field straddled past that window. The 2^ofc leading term survives, leaving an in-bounds but too-small offset that splices data from the wrong position: output length stays correct, no error is thrown, and decompress() and Decompress corrupt identically. Read the fifth byte when the field needs it. Also adds streaming regression tests for the issue #19 scenario (embedded frames, exhaustive chunk-split sweeps, never-final mode, multi-frame) plus an opt-in large-offset test (FZSTD_BIG_TESTS=1), and updates the README note about the former 2^25 backreference limit. --- README.md | 2 +- src/index.ts | 2 + tests/README.md | 17 +++- tests/streaming_regression_test.ts | 148 +++++++++++++++++++++++++++++ 4 files changed, 165 insertions(+), 4 deletions(-) create mode 100644 tests/streaming_regression_test.ts diff --git a/README.md b/README.md index 22d9aea..101bb6d 100644 --- a/README.md +++ b/README.md @@ -94,4 +94,4 @@ Unlike my Zlib implementation [`fflate`](https://github.com/101arrowz/fflate), W - Some WASM ports cannot operate without being provided the decompressed size of the data in advance. `fzstd` decides how much memory to allocate from the frame headers. - `fzstd` is absolutely tiny: at **8kB minified and 3.8kB after gzipping**, it's much smaller than most WASM implementations. -Please note that unlike the reference implementation, `fzstd` only supports a maximum backreference distance of 225 bytes. If you need to decompress files with an "ultra" compression level (20 or greater) AND your files can be above 32MB decompressed, `fzstd` may fail to decompress properly. Consider using a WebAssembly port for files this large. \ No newline at end of file +Backreference distances up to the maximum window size `fzstd` accepts (about 2GB) are fully supported, including frames produced with `--long` or "ultra" compression levels (20 or greater). Note that decompressing such frames requires allocating the entire window in memory, so a WebAssembly port may still be a better fit for very large files. \ No newline at end of file diff --git a/src/index.ts b/src/index.ts index 742c8ac..eff7f92 100644 --- a/src/index.ts +++ b/src/index.ts @@ -544,6 +544,8 @@ const rzb = (dat: Uint8Array, st: DZstdState, out?: Uint8Array) => { cbt = (spos -= ofc) >> 3; const ofp = 1 << ofc; let off = ofp + (((dat[cbt] | (dat[cbt + 1] << 8) | (dat[cbt + 2] << 16) | (dat[cbt + 3] << 24)) >>> (spos & 7)) & (ofp - 1)); + // 4 bytes only guarantee 25 bits; offsets past 2^25 can need a fifth + if ((spos & 7) + ofc > 32) off += (dat[cbt + 4] << (32 - (spos & 7))) & (ofp - 1); cbt = (spos -= mlb[mlc]) >> 3; let ml = mlbl[mlc] + (((dat[cbt] | (dat[cbt + 1] << 8) | (dat[cbt + 2] << 16)) >> (spos & 7)) & ((1 << mlb[mlc]) - 1)); cbt = (spos -= llb[llc]) >> 3; diff --git a/tests/README.md b/tests/README.md index 5204039..f61e95d 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,9 +1,20 @@ -# fzstd Deno tests +# fzstd tests -This folder contains test cases for fzstd that can be run with [Deno](https://deno.land/) +This folder contains test cases for fzstd. ## How to execute tests +Original cases, run with [Deno](https://deno.land/): + +```bash +deno test --no-check tests/simple_cases_test.ts +``` + +Streaming and large-offset regression cases, run with [Bun](https://bun.sh/): + ```bash -deno test --no-check +bun test tests/streaming_regression_test.ts + +# optional large-offset test (~400 MB RAM, needs zstd on PATH) +FZSTD_BIG_TESTS=1 bun test tests/streaming_regression_test.ts ``` diff --git a/tests/streaming_regression_test.ts b/tests/streaming_regression_test.ts new file mode 100644 index 0000000..efa1ffa --- /dev/null +++ b/tests/streaming_regression_test.ts @@ -0,0 +1,148 @@ +// Regression tests for streaming decompression (issue #19 scenario) and +// for backreference offsets above 2^25 (silent corruption in rzb). +// Run with: bun test tests/streaming_regression_test.ts +// The embedded frames are real zstd CLI output; expected payloads are +// regenerated in JS, so no zstd binary is needed for the core tests. +import { describe, it, expect } from 'bun:test'; +import * as fzstd from '../src/index.ts'; + +const hex = (s: string) => new Uint8Array(s.match(/../g).map(b => parseInt(b, 16))); +const enc = new TextEncoder(); + +// "hello world " * 11000 (= 132000 B: blocks of 131072 + 928), zstd -3, +// single-segment frame WITH xxh64 checksum. Same bytes as Go stdlib fixture +// testdata/f2a8e35c.helloworld-11000x.zst (golang.org/cl/531255). +const HW11000_CHECK = hex( + '28b52ffda4a0030200ac00006868656c6c6f20776f726c6420680100f0ffcfcb173d00000001009d5f1520d2e9d158' +); +// Same payload, zstd -3 --no-check via stdin: window-descriptor frame, +// no content size, NO checksum. Frames like this ended exactly at the last +// block and starved the pre-0.1.1 push() gate (fixed in 807a854). +const HW11000_NOCHECK = hex( + '28b52ffd0058a400006068656c6c6f20776f726c64200100f1ffcf4b12450000087201009c2b2004' +); +// "hello world " * 21845 (= 262140 B: 131072 + 131068), zstd -3 --no-check. +const HW21845_NOCHECK = hex( + '28b52ffd0058a400006068656c6c6f20776f726c64200100f1ffcf4b124d000008720100f8ff391002' +); + +const CASES: [string, Uint8Array, Uint8Array][] = [ + ['hw11000+checksum', HW11000_CHECK, enc.encode('hello world '.repeat(11000))], + ['hw11000 no-check', HW11000_NOCHECK, enc.encode('hello world '.repeat(11000))], + ['hw21845 no-check', HW21845_NOCHECK, enc.encode('hello world '.repeat(21845))], +]; + +// Streams data into a Decompress instance and returns the joined output. +// mode: 'final-on-last' | 'empty-final' | 'never-final' +const stream = (chunks: Uint8Array[], mode: string) => { + const parts: Uint8Array[] = []; + let sawFinal = false; + const d = new fzstd.Decompress((c, f) => { parts.push(c); if (f) sawFinal = true; }); + for (let i = 0; i < chunks.length; ++i) { + d.push(chunks[i], mode == 'final-on-last' && i == chunks.length - 1); + } + if (mode == 'empty-final') d.push(new Uint8Array(0), true); + const out = new Uint8Array(parts.reduce((s, p) => s + p.length, 0)); + let o = 0; + for (const p of parts) { out.set(p, o); o += p.length; } + return { out, sawFinal }; +}; + +describe('issue #19 scenario: >128KiB output via streaming', () => { + for (const [name, zst, want] of CASES) { + it(`${name}: one-shot`, () => { + expect(fzstd.decompress(zst)).toEqual(want); + }); + + it(`${name}: single push with final=true`, () => { + const { out, sawFinal } = stream([zst], 'final-on-last'); + expect(out).toEqual(want); + expect(sawFinal).toBe(true); + }); + + it(`${name}: push all, then empty final push`, () => { + const { out, sawFinal } = stream([zst], 'empty-final'); + expect(out).toEqual(want); + expect(sawFinal).toBe(true); + }); + + it(`${name}: push without ever sending final`, () => { + // The #19 reporter's adapter never pushed final; all data must still + // be delivered through ondata. + expect(stream([zst], 'never-final').out).toEqual(want); + }); + + it(`${name}: every two-chunk split, all final modes`, () => { + // Splits at/after the last block's start starved fzstd <= 0.1.0 on + // checksum-less frames: silent truncation to 131072 bytes without + // final, "unexpected EOF" with final. + for (let s = 1; s < zst.length; ++s) { + const chunks = [zst.subarray(0, s), zst.subarray(s)]; + for (const mode of ['final-on-last', 'empty-final', 'never-final']) { + expect(stream(chunks, mode).out).toEqual(want); + } + } + }); + + it(`${name}: one byte per push`, () => { + const chunks: Uint8Array[] = []; + for (let i = 0; i < zst.length; ++i) chunks.push(zst.subarray(i, i + 1)); + expect(stream(chunks, 'empty-final').out).toEqual(want); + }); + } +}); + +describe('multi-frame streams', () => { + const a = CASES[0], b = CASES[1]; + const joined = new Uint8Array(a[1].length + b[1].length); + joined.set(a[1], 0), joined.set(b[1], a[1].length); + const want = new Uint8Array(a[2].length + b[2].length); + want.set(a[2], 0), want.set(b[2], a[2].length); + + it('decodes both frames one-shot', () => { + expect(fzstd.decompress(joined)).toEqual(want); + }); + + it('decodes both frames streaming at every two-chunk split', () => { + for (let s = 1; s < joined.length; ++s) { + const chunks = [joined.subarray(0, s), joined.subarray(s)]; + expect(stream(chunks, 'empty-final').out).toEqual(want); + } + }); +}); + +// Backreference offsets >= 2^26 + 2^25 used to corrupt silently (the 4-byte +// offset read supplies only 32 - (spos & 7) bits). Needs ~230 MB of buffers +// and the zstd CLI, so it is opt-in: FZSTD_BIG_TESTS=1 bun test ... +const bigTest = process.env.FZSTD_BIG_TESTS ? it : it.skip; +describe('offsets beyond 2^25 (opt-in large test)', () => { + bigTest('decodes a --long=27 frame with ~100MB match distances', async () => { + const { spawnSync } = await import('child_process'); + const probe = spawnSync('zstd', ['--version']); + if (probe.error) throw new Error('zstd CLI not found on PATH; cannot run big offset test'); + // 100 MiB of seeded LCG noise + 500 short copies scattered from near the + // start: each copy needs a fresh ~100 MiB offset (no repeat-offset reuse), + // so offset code 26 gets read at many bit alignments, some straddling the + // 4-byte window that pre-fix code was limited to. + const M = 1 << 20; + const size = 100 * M; + const buf = new Uint8Array(size + 500 * 4000); + // splitmix32: unlike an LCG, has no short-period low/mid bits that zstd + // would match at small offsets + for (let i = 0, z = 0; i < size; ++i) { + z = (z + 0x9E3779B9) | 0; + let x = z; + x = Math.imul(x ^ (x >>> 16), 0x21F0AAAD); + x = Math.imul(x ^ (x >>> 15), 0x735A2D97); + buf[i] = x ^ (x >>> 15); + } + for (let i = 0; i < 500; ++i) { + buf.set(buf.subarray(i * 4096, i * 4096 + 4000), size + i * 4000); + } + const z = spawnSync('zstd', ['-1', '--long=27', '-c'], { input: buf, maxBuffer: 1 << 30 }); + if (z.status !== 0) throw new Error('zstd failed: ' + z.stderr); + const out = fzstd.decompress(new Uint8Array(z.stdout)); + expect(out.length).toBe(buf.length); + expect(Buffer.compare(out, buf)).toBe(0); + }, 300000); +}); From 15007f33eb67068bf48210f88d282391f5effe1f Mon Sep 17 00:00:00 2001 From: Ernesto Serrano Date: Sat, 11 Jul 2026 12:57:42 +0100 Subject: [PATCH 2/2] Harden large-offset decoding and clarify limits Reject offset codes outside the decoder's signed 31-bit range, reuse the bit alignment and mask in the sequence loop, and keep the fifth-byte read limited to fields that actually cross the 32-bit window. Also narrow the README claims to the support actually provided and correct the large regression test's threshold, generator name, and measured memory requirement. --- README.md | 2 +- src/index.ts | 9 +++++---- tests/README.md | 2 +- tests/streaming_regression_test.ts | 12 ++++++------ 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 101bb6d..82c4b72 100644 --- a/README.md +++ b/README.md @@ -94,4 +94,4 @@ Unlike my Zlib implementation [`fflate`](https://github.com/101arrowz/fflate), W - Some WASM ports cannot operate without being provided the decompressed size of the data in advance. `fzstd` decides how much memory to allocate from the frame headers. - `fzstd` is absolutely tiny: at **8kB minified and 3.8kB after gzipping**, it's much smaller than most WASM implementations. -Backreference distances up to the maximum window size `fzstd` accepts (about 2GB) are fully supported, including frames produced with `--long` or "ultra" compression levels (20 or greater). Note that decompressing such frames requires allocating the entire window in memory, so a WebAssembly port may still be a better fit for very large files. \ No newline at end of file +Backreference distances are decoded up to the maximum frame window accepted by `fzstd` (about 2 GB). Frames that use `--long` or ultra compression levels can therefore be decoded when their offsets stay within that limit. Very large frames remain subject to JavaScript typed-array, memory, and frame-size limits, so a WebAssembly implementation may still be a better fit for multi-hundred-megabyte inputs. \ No newline at end of file diff --git a/src/index.ts b/src/index.ts index eff7f92..e2dcfd6 100644 --- a/src/index.ts +++ b/src/index.ts @@ -541,11 +541,12 @@ const rzb = (dat: Uint8Array, st: DZstdState, out?: Uint8Array) => { const ofc = oct.s[ost]; const obtr = oct.n[ost]; + if (ofc > 30) err(4); cbt = (spos -= ofc) >> 3; - const ofp = 1 << ofc; - let off = ofp + (((dat[cbt] | (dat[cbt + 1] << 8) | (dat[cbt + 2] << 16) | (dat[cbt + 3] << 24)) >>> (spos & 7)) & (ofp - 1)); - // 4 bytes only guarantee 25 bits; offsets past 2^25 can need a fifth - if ((spos & 7) + ofc > 32) off += (dat[cbt + 4] << (32 - (spos & 7))) & (ofp - 1); + const sh = spos & 7, ofp = 1 << ofc, ofm = ofp - 1; + let off = ofp + (((dat[cbt] | (dat[cbt + 1] << 8) | (dat[cbt + 2] << 16) | (dat[cbt + 3] << 24)) >>> sh) & ofm); + // Four bytes only guarantee 25 bits; large offsets can need a fifth. + if (sh + ofc > 32) off += (dat[cbt + 4] << (32 - sh)) & ofm; cbt = (spos -= mlb[mlc]) >> 3; let ml = mlbl[mlc] + (((dat[cbt] | (dat[cbt + 1] << 8) | (dat[cbt + 2] << 16)) >> (spos & 7)) & ((1 << mlb[mlc]) - 1)); cbt = (spos -= llb[llc]) >> 3; diff --git a/tests/README.md b/tests/README.md index f61e95d..3e7b524 100644 --- a/tests/README.md +++ b/tests/README.md @@ -15,6 +15,6 @@ Streaming and large-offset regression cases, run with [Bun](https://bun.sh/): ```bash bun test tests/streaming_regression_test.ts -# optional large-offset test (~400 MB RAM, needs zstd on PATH) +# Optional large-offset test (up to ~800 MB peak RAM, needs zstd on PATH). FZSTD_BIG_TESTS=1 bun test tests/streaming_regression_test.ts ``` diff --git a/tests/streaming_regression_test.ts b/tests/streaming_regression_test.ts index efa1ffa..6c046fb 100644 --- a/tests/streaming_regression_test.ts +++ b/tests/streaming_regression_test.ts @@ -1,5 +1,5 @@ // Regression tests for streaming decompression (issue #19 scenario) and -// for backreference offsets above 2^25 (silent corruption in rzb). +// for large backreference offsets that cross a 4-byte bit-reading window. // Run with: bun test tests/streaming_regression_test.ts // The embedded frames are real zstd CLI output; expected payloads are // regenerated in JS, so no zstd binary is needed for the core tests. @@ -111,16 +111,16 @@ describe('multi-frame streams', () => { }); }); -// Backreference offsets >= 2^26 + 2^25 used to corrupt silently (the 4-byte -// offset read supplies only 32 - (spos & 7) bits). Needs ~230 MB of buffers -// and the zstd CLI, so it is opt-in: FZSTD_BIG_TESTS=1 bun test ... +// The first affected distance is 2^26 + 2^25 - 3 (about 96 MiB): the 4-byte +// offset read supplies only 32 - (spos & 7) bits. The test can use up to +// ~800 MB peak RAM and needs zstd on PATH, so it is opt-in. const bigTest = process.env.FZSTD_BIG_TESTS ? it : it.skip; -describe('offsets beyond 2^25 (opt-in large test)', () => { +describe('large backreference offsets (opt-in large test)', () => { bigTest('decodes a --long=27 frame with ~100MB match distances', async () => { const { spawnSync } = await import('child_process'); const probe = spawnSync('zstd', ['--version']); if (probe.error) throw new Error('zstd CLI not found on PATH; cannot run big offset test'); - // 100 MiB of seeded LCG noise + 500 short copies scattered from near the + // 100 MiB of seeded SplitMix32 noise + 500 short copies scattered from near the // start: each copy needs a fresh ~100 MiB offset (no repeat-offset reuse), // so offset code 26 gets read at many bit alignments, some straddling the // 4-byte window that pre-fix code was limited to.